From 50ef0402c649bafc046c8ebb185b747696f536c1 Mon Sep 17 00:00:00 2001 From: stoney420 Date: Sun, 29 Jun 2025 05:30:52 +0200 Subject: [PATCH] Update .forgejo/workflows/generate-m3u.yml --- .forgejo/workflows/generate-m3u.yml | 432 +++++----------------------- 1 file changed, 68 insertions(+), 364 deletions(-) diff --git a/.forgejo/workflows/generate-m3u.yml b/.forgejo/workflows/generate-m3u.yml index b24c7dc..1983088 100644 --- a/.forgejo/workflows/generate-m3u.yml +++ b/.forgejo/workflows/generate-m3u.yml @@ -1,12 +1,10 @@ -name: Generate M3U Playlist - Enhanced Country Detection +name: Generate M3U Playlist with Auto-Organization + on: push: branches: - main workflow_dispatch: - schedule: - # Run daily at 2 AM UTC to refresh playlist - - cron: '0 2 * * *' jobs: build-and-organize: @@ -14,390 +12,100 @@ jobs: steps: - name: Checkout Repository uses: actions/checkout@v4 - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.11' - + - name: Configure Git run: | git config --local user.email "actions@forgejo.plainrock127.xyz" git config --local user.name "IPTV Playlist Bot" - - - name: Create Required Directories + + - name: Setup Python Environment run: | - echo "๐Ÿ—๏ธ Setting up directory structure..." - mkdir -p reports/daily reports/archive backups logs - echo "โœ… Directory structure created" + echo "Setting up Python environment..." + python3 --version + echo "Setting up directories..." + mkdir -p reports/daily reports/logs backups config + echo "Setup completed" - - name: Install Python Dependencies + - name: Check Import File run: | - echo "๐Ÿ“ฆ Installing Python dependencies..." - python3 -m pip install --upgrade pip - python3 -m pip install requests aiohttp asyncio beautifulsoup4 - echo "โœ… Dependencies installed" - - - name: Verify Import File - run: | - echo "๐Ÿ“‹ Checking import file..." + echo "Checking import file..." if [ -f bulk_import.m3u ]; then LINES=$(wc -l < bulk_import.m3u) - echo "๐Ÿ“บ Found bulk_import.m3u with $LINES lines" - # Show first few lines for debugging - echo "๐Ÿ“„ First 10 lines of import file:" - head -10 bulk_import.m3u + echo "Found bulk_import.m3u with $LINES lines" else - echo "โš ๏ธ No bulk_import.m3u found, creating empty file" + echo "Creating empty bulk_import.m3u" echo '#EXTM3U' > bulk_import.m3u - echo '# Add your M3U content here' >> bulk_import.m3u + fi + + - name: Verify Scripts Directory + run: | + echo "Checking scripts directory..." + if [ -d scripts ]; then + echo "Scripts directory exists" + ls -la scripts/ + else + echo "Scripts directory not found!" + exit 1 fi - name: Run Playlist Generation run: | - echo "๐Ÿš€ Starting playlist generation..." + echo "Running playlist generation..." cd scripts - # Make the script executable - chmod +x generate_playlist.py - # Run the main script with enhanced logging - python3 generate_playlist.py 2>&1 | tee ../logs/generation.log - if [ $? -eq 0 ]; then - echo "โœ… Playlist generation completed successfully" + if [ -f generate_playlist.py ]; then + echo "Found generate_playlist.py, executing..." + python3 generate_playlist.py + echo "Playlist generation completed" else - echo "โŒ Playlist generation failed" - echo "๐Ÿ“‹ Last 20 lines of log:" - tail -20 ../logs/generation.log + echo "Error: generate_playlist.py not found" exit 1 fi - - name: Validate Generated Files + - name: Create Simple Report run: | - echo "๐Ÿ” Validating generated files..." - # Check if playlist was generated + echo "Creating report..." if [ -f playlist.m3u ]; then CHANNELS=$(grep -c "^#EXTINF" playlist.m3u || echo "0") - echo "โœ… Generated playlist.m3u with $CHANNELS channels" - # Show playlist statistics - GROUPS=$(grep -o 'group-title="[^"]*"' playlist.m3u | sort -u | wc -l || echo "0") - echo "๐Ÿ“Š Organized into $GROUPS groups" + echo "Found $CHANNELS channels in playlist" + + DATE=$(date +%Y%m%d_%H%M%S) + REPORT="reports/daily/report_$DATE.md" + + echo "# Playlist Report" > "$REPORT" + echo "Generated: $(date)" >> "$REPORT" + echo "Channels: $CHANNELS" >> "$REPORT" + + echo "Report created: $REPORT" else - echo "โš ๏ธ No playlist.m3u generated" - fi - # Check channels.txt - if [ -f channels.txt ]; then - CHANNEL_BLOCKS=$(grep -c "^Stream name =" channels.txt || echo "0") - echo "โœ… channels.txt contains $CHANNEL_BLOCKS channel records" - else - echo "โš ๏ธ No channels.txt found" + echo "No playlist found" fi - - name: Generate Comprehensive Report + - name: Clean Old Reports run: | - echo "๐Ÿ“Š Creating comprehensive report..." - DATE=$(date +%Y%m%d_%H%M%S) - REPORT="reports/daily/report_$DATE.md" - - echo "# ๐Ÿ“บ IPTV Playlist Generation Report" > "$REPORT" - echo "**Generated:** $(date)" >> "$REPORT" - echo "" >> "$REPORT" - - # Playlist statistics - if [ -f playlist.m3u ]; then - CHANNELS=$(grep -c "^#EXTINF" playlist.m3u || echo "0") - GROUPS=$(grep -o 'group-title="[^"]*"' playlist.m3u | sort -u | wc -l || echo "0") - - echo "## ๐Ÿ“Š Statistics" >> "$REPORT" - echo "- **Total Channels:** $CHANNELS" >> "$REPORT" - echo "- **Country Groups:** $GROUPS" >> "$REPORT" - echo "" >> "$REPORT" - - # List top 10 groups - echo "## ๐ŸŒ Top Channel Groups" >> "$REPORT" - grep -o 'group-title="[^"]*"' playlist.m3u | sed 's/group-title="//;s/"//' | sort | uniq -c | sort -nr | head -10 | while read count group; do - echo "- **$group:** $count channels" >> "$REPORT" - done - else - echo "## โš ๏ธ No playlist generated" >> "$REPORT" - fi - - echo "" >> "$REPORT" - echo "---" >> "$REPORT" - echo "*Report generated automatically by Forgejo Actions*" >> "$REPORT" - - echo "โœ… Report created: "$REPORT"" - - - name: Health Check (Optional) - continue-on-error: true - run: | - echo "๐Ÿฅ Running health check on sample channels..." - if [ -f playlist.m3u ]; then - # Test first 5 channels only to avoid long execution times - echo "Testing first 5 channels for connectivity..." - head -20 playlist.m3u | grep -E "^(#EXTINF|http)" | head -10 - fi - - - name: Cleanup and Archive - run: | - echo "๐Ÿงน Running cleanup and archiving tasks..." - - # Clean Old Reports - echo "Cleaning old daily reports..." - REPORT_DIR="reports/daily" - if [ -d "$REPORT_DIR" ]; then - cd "$REPORT_DIR" - if ls *.md >/dev/null 2>&1; then - COUNT=$(ls *.md | wc -l) - echo "Found $COUNT reports" - if [ "$COUNT" -gt 3 ]; then - echo "Removing excess reports..." - ls -t *.md | tail -n +4 | xargs rm -f - echo "Cleanup done" - fi - else - echo "No reports to clean in ${REPORT_DIR}" + echo "Cleaning old reports..." + cd reports/daily + if ls *.md >/dev/null 2>&1; then + COUNT=$(ls *.md | wc -l) + echo "Found $COUNT reports" + if [ "$COUNT" -gt 3 ]; then + echo "Removing excess reports..." + ls -t *.md | tail -n +4 | xargs rm -f + echo "Cleanup done" fi - cd - > /dev/null # Go back to original directory else - echo "Daily reports directory ${REPORT_DIR} not found." + echo "No reports to clean" fi + cd ../.. - # Reset Import File - echo "Resetting bulk_import.m3u..." + - name: Reset Import File + run: | + echo "Resetting import file..." echo '#EXTM3U' > bulk_import.m3u echo "Import file reset" - - name: Create Country Fix Script + - name: Commit Changes run: | - echo "๐Ÿ”ง Creating country classification fix script..." - cat > country_fix.py << 'EOF' -#!/usr/bin/env python3 -"""Enhanced Country Detection Fix""" -import re -import shutil -from datetime import datetime -def detect_country_enhanced(channel_name, epg_id="", logo_url="", stream_url=""): - text = f"{channel_name} {epg_id} {logo_url} {stream_url}".lower() - # EPG ID suffix detection (highest priority) - if ".ca" in epg_id.lower(): - return "๐Ÿ‡จ๐Ÿ‡ฆ Canada" - elif ".us" in epg_id.lower(): - return "๐Ÿ‡บ๐Ÿ‡ธ United States" - elif ".uk" in epg_id.lower(): - return "๐Ÿ‡ฌ๐Ÿ‡ง United Kingdom" - elif ".ph" in epg_id.lower(): - return "๐Ÿ‡ต๐Ÿ‡ญ Philippines" - elif ".au" in epg_id.lower(): - return "๐Ÿ‡ฆ๐Ÿ‡บ Australia" - elif ".jp" in epg_id.lower(): - return "๐Ÿ‡ฏ๐Ÿ‡ต Japan" - channel_lower = channel_name.lower() - # Specific fixes for misclassified channels - if any(x in channel_lower for x in ["tsn 1", "tsn 2", "tsn 3", "tsn 4", "tsn 5", "cbc news toronto"]): - return "๐Ÿ‡จ๐Ÿ‡ฆ Canada" - if any(x in channel_lower for x in ["tv land", "tvland", "we tv", "wetv", "all weddings we tv"]): - return "๐Ÿ‡บ๐Ÿ‡ธ United States" - if "come dine with me" in channel_lower: - return "๐Ÿ‡ฌ๐Ÿ‡ง United Kingdom" - if "all tv" in channel_lower and "brandlogo.org" in logo_url: - return "๐Ÿ‡ฌ๐Ÿ‡ง United Kingdom" - if any(x in channel_lower for x in ["anc global", "anc ph"]): - return "๐Ÿ‡ต๐Ÿ‡ญ Philippines" - if "animax" in channel_lower: - return "๐Ÿ‡ฏ๐Ÿ‡ต Japan" - # Platform-based detection - if "pluto.tv" in text or "images.pluto.tv" in text: - pluto_overrides = { - "cbc news toronto": "๐Ÿ‡จ๐Ÿ‡ฆ Canada", - "come dine with me": "๐Ÿ‡ฌ๐Ÿ‡ง United Kingdom" - } - for pattern, country in pluto_overrides.items(): - if pattern in channel_lower: - return country - return "๐Ÿ‡บ๐Ÿ‡ธ United States" - if "plex.tv" in text: - return "๐Ÿ‡บ๐Ÿ‡ธ United States" - # Country pattern matching - patterns = { - "๐Ÿ‡บ๐Ÿ‡ธ United States": ["usa", "us ", "america", "cbs", "nbc", "abc", "fox", "espn", "cnn", "amc", "mtv", "comedy central", "nickelodeon", "disney", "hgtv", "syfy", "bravo", "tlc", "lifetime", "paramount", "weather channel", "tmz", "wgn"], - "๐Ÿ‡จ๐Ÿ‡ฆ Canada": ["canada", "canadian", "cbc", "ctv", "global", "tsn", "sportsnet", "w network", "much", "teletoon"], - "๐Ÿ‡ฌ๐Ÿ‡ง United Kingdom": ["uk", "british", "bbc", "itv", "sky", "channel 4", "channel 5", "dave", "quest", "bt sport", "premier league"], - "๐Ÿ‡ต๐Ÿ‡ญ Philippines": ["philippines", "filipino", "abs-cbn", "gma", "anc", "cnn philippines"], - "๐Ÿ‡ฆ๐Ÿ‡บ Australia": ["australia", "australian", "abc australia", "nine network", "seven network", "ten network"], - "๐Ÿ‡ฏ๐Ÿ‡ต Japan": ["japan", "japanese", "nhk", "fuji tv", "animax"], - "๐Ÿ‡ฎ๐Ÿ‡ณ India": ["india", "indian", "hindi", "zee", "star", "sony", "colors"], - "๐Ÿ‡ฉ๐Ÿ‡ช Germany": ["germany", "german", "ard", "zdf", "rtl", "sat.1", "pro7"], - "๐Ÿ‡ซ๐Ÿ‡ท France": ["france", "french", "tf1", "france 2", "m6", "canal+"], - "๐Ÿ‡ช๐Ÿ‡ธ Spain": ["spain", "spanish", "antena 3", "telecinco", "tve"], - "๐Ÿ‡ฎ๐Ÿ‡น Italy": ["italy", "italian", "rai", "mediaset", "canale 5"], - "๐Ÿ‡ณ๐Ÿ‡ฑ Netherlands": ["netherlands", "dutch", "npo", "rtl 4"], - "๐Ÿ‡ง๐Ÿ‡ท Brazil": ["brazil", "brazilian", "globo", "sbt", "record"], - "๐Ÿ‡ฒ๐Ÿ‡ฝ Mexico": ["mexico", "mexican", "televisa", "tv azteca"], - "๐Ÿ‡ท๐Ÿ‡บ Russia": ["russia", "russian", "ะฟะตั€ะฒั‹ะน", "ั€ะพััะธั", "ะฝั‚ะฒ"] - } - for country, keywords in patterns.items(): - if any(keyword in text for keyword in keywords): - return country - return "๐ŸŒ International" -def fix_channels(): - backup_name = f"channels_backup_{datetime.now().strftime('%Y%m%d_%H%M%S')}.txt" - shutil.copy2("channels.txt", backup_name) - print(f"๐Ÿ“‹ Created backup: {backup_name}") - with open("channels.txt", 'r', encoding='utf-8') as f: - content = f.read() - blocks = content.split('\n\n') - fixed_channels = [] - changes_made = 0 - country_stats = {} - for block in blocks: - if not block.strip(): - continue - lines = block.strip().split('\n') - channel_data = {} - for line in lines: - if '=' in line: - key, value = line.split('=', 1) - channel_data[key.strip()] = value.strip() - if not channel_data: - continue - current_group = channel_data.get('Group', 'Uncategorized') - stream_name = channel_data.get('Stream name', '') - epg_id = channel_data.get('EPG id', '') - logo = channel_data.get('Logo', '') - stream_url = channel_data.get('Stream URL', '') - detected_country = detect_country_enhanced(stream_name, epg_id, logo, stream_url) - if current_group != detected_country: - print(f"๐Ÿ”„ Fix: '{stream_name}' {current_group} โ†’ {detected_country}") - channel_data['Group'] = detected_country - changes_made += 1 - country_stats[channel_data['Group']] = country_stats.get(channel_data['Group'], 0) + 1 - fixed_block = [] - for key in ['Group', 'Stream name', 'Logo', 'EPG id', 'Stream URL']: - fixed_block.append(f"{key} = {channel_data.get(key, '')}") - fixed_channels.append('\n'.join(fixed_block)) - with open("channels.txt", 'w', encoding='utf-8') as f: - f.write('\n\n'.join(fixed_channels)) - print(f"โœ… Changes made: {changes_made}") - print(f"๐Ÿ“บ Total channels: {len(fixed_channels)}") - # Show top countries - sorted_countries = sorted(country_stats.items(), key=lambda x: x[1], reverse=True) - print("๐ŸŒ Top Countries:") - for country, count in sorted_countries[:10]: - percentage = (count / len(fixed_channels) * 100) if len(fixed_channels) > 0 else 0 - print(f" {country}: {count} ({percentage:.1f}%)") - # Calculate success rate - unclassified = country_stats.get('Uncategorized', 0) + country_stats.get('๐ŸŒ International', 0) - success_rate = ((len(fixed_channels) - unclassified) / len(fixed_channels) * 100) if len(fixed_channels) > 0 else 0 - print(f"๐ŸŽฏ Detection Success Rate: {success_rate:.1f}%") - return len(fixed_channels), changes_made -if __name__ == "__main__": - total, changes = fix_channels() - print(f"๐ŸŽ‰ Country fix completed! {changes} channels reclassified.") -EOF - chmod +x country_fix.py - echo "โœ… Country fix script created" - - - name: Run Country Classification Fix - run: | - echo "๐Ÿ”ง Running enhanced country classification fix..." - python3 country_fix.py - if [ $? -eq 0 ]; then - echo "โœ… Country classification fix completed successfully" - else - echo "โŒ Country classification fix failed" - exit 1 - fi - - - name: Generate Enhanced M3U Playlist - run: | - echo "๐Ÿ“บ Generating enhanced M3U playlist..." - # Create a temporary Python script for playlist generation - cat > generate_enhanced_playlist.py << 'EOF' -#!/usr/bin/env python3 -import sys - -print('๐Ÿš€ Generating M3U from fixed channels...') -# Read channels.txt -with open('channels.txt', 'r', encoding='utf-8') as f: - content = f.read() -blocks = content.split('\n\n') -# Generate M3U -with open('playlist.m3u', 'w', encoding='utf-8') as f: - f.write('#EXTM3U\n') - valid_channels = 0 - for block in blocks: - if not block.strip(): - continue - lines = block.strip().split('\n') - channel_data = {} - for line in lines: - if '=' in line: - key, value = line.split('=', 1) - channel_data[key.strip()] = value.strip() - stream_name = channel_data.get('Stream name', '') - group = channel_data.get('Group', '') - logo = channel_data.get('Logo', '') - epg_id = channel_data.get('EPG id', '') - url = channel_data.get('Stream URL', '') - if stream_name and url: - f.write(f'#EXTINF:-1 group-title=\"{group}\"') - if logo: - f.write(f' tvg-logo=\"{logo}\"') - if epg_id: - f.write(f' tvg-id=\"{epg_id}\"') - f.write(f',{stream_name}\n') - f.write(f'{url}\n') - valid_channels += 1 -print(f'โœ… Generated playlist.m3u with {valid_channels} channels') -EOF - # Make the temporary script executable and run it - chmod +x generate_enhanced_playlist.py - python3 generate_enhanced_playlist.py - - - name: Validate Results and Generate Report - run: | - echo "๐Ÿ” Validating results..." - if [ -f channels.txt ]; then - CHANNEL_COUNT=$(grep -c "^Stream name =" channels.txt || echo "0") - echo "โœ… channels.txt: $CHANNEL_COUNT channels" - else - echo "โŒ channels.txt not found" - exit 1 - fi - if [ -f playlist.m3u ]; then - PLAYLIST_COUNT=$(grep -c "^#EXTINF" playlist.m3u || echo "0") - echo "โœ… playlist.m3u: $PLAYLIST_COUNT channels" - else - echo "โŒ playlist.m3u not found" - exit 1 - fi - - # Generate comprehensive report - DATE=$(date +%Y%m%d_%H%M%S) - REPORT="reports/daily/enhanced_report_$DATE.md" - - echo "# ๐ŸŒ Enhanced Country Detection Report" > "$REPORT" - echo "**Generated:** $(date)" >> "$REPORT" - echo "" >> "$REPORT" - echo "## ๐Ÿ“Š Statistics" >> "$REPORT" - echo "- **Total Channels:** $CHANNEL_COUNT" >> "$REPORT" - echo "- **Playlist Channels:** $PLAYLIST_COUNT" >> "$REPORT" - echo "" >> "$REPORT" - - # Count countries in playlist - if [ -f playlist.m3u ]; then - echo "## ๐ŸŒ Country Distribution" >> "$REPORT" - grep -o 'group-title="[^"]*"' playlist.m3u | sed 's/group-title="//;s/"//' | sort | uniq -c | sort -nr | head -15 | while read count country; do - echo "- **$country:** $count channels" >> "$REPORT" - done - fi - echo "" >> "$REPORT" - echo "---" >> "$REPORT" - echo "*Enhanced country detection with 99%+ accuracy*" >> "$REPORT" - echo "๐Ÿ“Š Report created: "$REPORT"" - - - name: Commit Enhanced Results - run: | - echo "๐Ÿ’พ Committing enhanced results..." + echo "Committing changes..." git add . if git diff --staged --quiet; then echo "No changes to commit" @@ -406,21 +114,17 @@ EOF if [ -f playlist.m3u ]; then CHANNELS=$(grep -c "^#EXTINF" playlist.m3u || echo "0") fi - git commit -m "๐ŸŒ Enhanced country detection: $CHANNELS channels with improved accuracy - $(date '+%Y-%m-%d %H:%M')" + git commit -m "๐Ÿ“บ Updated playlist with $CHANNELS channels - $(date '+%Y-%m-%d %H:%M')" git push - echo "โœ… Enhanced results committed" + echo "Changes committed" fi - - name: Final Summary + - name: Summary run: | - echo "๐ŸŽ‰ ENHANCED COUNTRY DETECTION COMPLETE!" - echo "==================================================" + echo "=== WORKFLOW COMPLETE ===" if [ -f playlist.m3u ]; then CHANNELS=$(grep -c "^#EXTINF" playlist.m3u || echo "0") - COUNTRIES=$(grep -o 'group-title="[^"]*"' playlist.m3u | sed 's/group-title="//;s/"//' | sort -u | wc -l || echo "0") - echo "โœ… Generated "$CHANNELS" channels across "$COUNTRIES" countries" - echo "๐ŸŽฏ Enhanced detection fixes major classification issues" - echo "๐ŸŒ Canadian, US, UK, Philippines channels now correctly classified" + echo "โœ… Playlist has $CHANNELS channels" fi - echo "๐Ÿ“‹ Backup files created for safety" - echo "๐Ÿ“Š Detailed reports available in reports/daily/" + echo "โœ… Reports updated" + echo "โœ… Repository cleaned" \ No newline at end of file