Delete scripts/source_scraper.py
Some checks are pending
Generate M3U Playlist with Auto-Organization / build-and-organize (push) Waiting to run
Some checks are pending
Generate M3U Playlist with Auto-Organization / build-and-organize (push) Waiting to run
This commit is contained in:
parent
f9612e7acc
commit
482d4cd7f1
1 changed files with 0 additions and 294 deletions
|
|
@ -1,294 +0,0 @@
|
||||||
# scripts/source_scraper.py
|
|
||||||
"""
|
|
||||||
Automated Channel Discovery for IPTV Playlist Generator
|
|
||||||
Integrates seamlessly with existing architecture
|
|
||||||
"""
|
|
||||||
|
|
||||||
import requests
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import logging
|
|
||||||
from datetime import datetime
|
|
||||||
from typing import List, Dict, Set
|
|
||||||
import re
|
|
||||||
from urllib.parse import urlparse
|
|
||||||
|
|
||||||
class SourceScraper:
|
|
||||||
def __init__(self):
|
|
||||||
self.setup_logging()
|
|
||||||
self.load_config()
|
|
||||||
self.discovered_channels = []
|
|
||||||
self.source_stats = {}
|
|
||||||
|
|
||||||
def setup_logging(self):
|
|
||||||
"""Setup logging consistent with existing system"""
|
|
||||||
logging.basicConfig(
|
|
||||||
level=logging.INFO,
|
|
||||||
format='%(asctime)s - %(levelname)s - %(message)s'
|
|
||||||
)
|
|
||||||
self.logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
def load_config(self):
|
|
||||||
"""Load discovery sources configuration"""
|
|
||||||
try:
|
|
||||||
with open('config/discovery_sources.json', 'r') as f:
|
|
||||||
self.config = json.load(f)
|
|
||||||
except FileNotFoundError:
|
|
||||||
# Create default config if it doesn't exist
|
|
||||||
self.config = {
|
|
||||||
"enabled": True,
|
|
||||||
"sources": [
|
|
||||||
{
|
|
||||||
"name": "IPTV-Org Main",
|
|
||||||
"url": "https://raw.githubusercontent.com/iptv-org/iptv/master/streams/",
|
|
||||||
"type": "github_directory",
|
|
||||||
"country_filter": ["us", "uk", "ca", "au"]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "Free-TV Collection",
|
|
||||||
"url": "https://raw.githubusercontent.com/Free-TV/IPTV/master/playlist.m3u8",
|
|
||||||
"type": "m3u_playlist",
|
|
||||||
"quality_filter": ["hd", "fhd", "4k"]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"filters": {
|
|
||||||
"min_quality": "sd",
|
|
||||||
"exclude_adult": True,
|
|
||||||
"max_channels_per_source": 100,
|
|
||||||
"require_country_detection": False
|
|
||||||
},
|
|
||||||
"rate_limiting": {
|
|
||||||
"delay_between_requests": 1.0,
|
|
||||||
"max_retries": 3
|
|
||||||
}
|
|
||||||
}
|
|
||||||
os.makedirs('config', exist_ok=True)
|
|
||||||
with open('config/discovery_sources.json', 'w') as f:
|
|
||||||
json.dump(self.config, f, indent=2)
|
|
||||||
|
|
||||||
def discover_from_m3u_url(self, source_info: Dict) -> List[str]:
|
|
||||||
"""Discover channels from M3U playlist URL"""
|
|
||||||
try:
|
|
||||||
self.logger.info(f"Discovering from M3U: {source_info['name']}")
|
|
||||||
|
|
||||||
response = requests.get(source_info['url'], timeout=30)
|
|
||||||
response.raise_for_status()
|
|
||||||
|
|
||||||
content = response.text
|
|
||||||
channels = []
|
|
||||||
|
|
||||||
# Parse M3U content
|
|
||||||
lines = content.split('\n')
|
|
||||||
current_extinf = None
|
|
||||||
|
|
||||||
for line in lines:
|
|
||||||
line = line.strip()
|
|
||||||
if line.startswith('#EXTINF:'):
|
|
||||||
current_extinf = line
|
|
||||||
elif line.startswith('http') and current_extinf:
|
|
||||||
# We have a complete channel entry
|
|
||||||
channels.append(f"{current_extinf}\n{line}")
|
|
||||||
current_extinf = None
|
|
||||||
|
|
||||||
self.logger.info(f"Found {len(channels)} channels from {source_info['name']}")
|
|
||||||
self.source_stats[source_info['name']] = len(channels)
|
|
||||||
|
|
||||||
return channels[:self.config['filters']['max_channels_per_source']]
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
self.logger.error(f"Error discovering from {source_info['name']}: {e}")
|
|
||||||
return []
|
|
||||||
|
|
||||||
def discover_from_github_directory(self, source_info: Dict) -> List[str]:
|
|
||||||
"""Discover channels from GitHub directory structure"""
|
|
||||||
try:
|
|
||||||
self.logger.info(f"Discovering from GitHub: {source_info['name']}")
|
|
||||||
|
|
||||||
base_url = source_info['url']
|
|
||||||
channels = []
|
|
||||||
|
|
||||||
# Try common country codes from your existing patterns
|
|
||||||
country_codes = source_info.get('country_filter', ['us', 'uk', 'ca', 'de', 'fr'])
|
|
||||||
|
|
||||||
for country in country_codes:
|
|
||||||
try:
|
|
||||||
url = f"{base_url}{country}.m3u"
|
|
||||||
response = requests.get(url, timeout=15)
|
|
||||||
|
|
||||||
if response.status_code == 200:
|
|
||||||
content = response.text
|
|
||||||
lines = content.split('\n')
|
|
||||||
current_extinf = None
|
|
||||||
|
|
||||||
for line in lines:
|
|
||||||
line = line.strip()
|
|
||||||
if line.startswith('#EXTINF:'):
|
|
||||||
current_extinf = line
|
|
||||||
elif line.startswith('http') and current_extinf:
|
|
||||||
channels.append(f"{current_extinf}\n{line}")
|
|
||||||
current_extinf = None
|
|
||||||
|
|
||||||
self.logger.info(f"Found {len(channels)} channels for {country}")
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
self.logger.debug(f"No channels found for {country}: {e}")
|
|
||||||
continue
|
|
||||||
|
|
||||||
self.source_stats[source_info['name']] = len(channels)
|
|
||||||
return channels[:self.config['filters']['max_channels_per_source']]
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
self.logger.error(f"Error discovering from GitHub {source_info['name']}: {e}")
|
|
||||||
return []
|
|
||||||
|
|
||||||
def filter_channels(self, channels: List[str]) -> List[str]:
|
|
||||||
"""Apply quality and content filters"""
|
|
||||||
if not self.config['filters']['exclude_adult']:
|
|
||||||
return channels
|
|
||||||
|
|
||||||
# Load adult keywords from existing config
|
|
||||||
try:
|
|
||||||
with open('config/patterns.json', 'r') as f:
|
|
||||||
patterns = json.load(f)
|
|
||||||
adult_keywords = patterns.get('adult_keywords', [])
|
|
||||||
except:
|
|
||||||
adult_keywords = ['xxx', 'adult', 'porn', '+18']
|
|
||||||
|
|
||||||
filtered = []
|
|
||||||
for channel in channels:
|
|
||||||
# Check if channel contains adult content
|
|
||||||
channel_lower = channel.lower()
|
|
||||||
if not any(keyword in channel_lower for keyword in adult_keywords):
|
|
||||||
filtered.append(channel)
|
|
||||||
|
|
||||||
self.logger.info(f"Filtered {len(channels) - len(filtered)} adult channels")
|
|
||||||
return filtered
|
|
||||||
|
|
||||||
def deduplicate_with_existing(self, new_channels: List[str]) -> List[str]:
|
|
||||||
"""Remove channels that already exist in channels.txt"""
|
|
||||||
if not os.path.exists('channels.txt'):
|
|
||||||
return new_channels
|
|
||||||
|
|
||||||
# Read existing channel URLs
|
|
||||||
existing_urls = set()
|
|
||||||
try:
|
|
||||||
with open('channels.txt', 'r', encoding='utf-8') as f:
|
|
||||||
content = f.read()
|
|
||||||
# Extract URLs from existing channels.txt
|
|
||||||
url_pattern = r'Stream URL\s*=\s*(.+)'
|
|
||||||
existing_urls = set(re.findall(url_pattern, content))
|
|
||||||
except Exception as e:
|
|
||||||
self.logger.warning(f"Could not read existing channels: {e}")
|
|
||||||
|
|
||||||
# Filter out duplicates
|
|
||||||
unique_channels = []
|
|
||||||
for channel in new_channels:
|
|
||||||
lines = channel.split('\n')
|
|
||||||
if len(lines) >= 2:
|
|
||||||
url = lines[1].strip()
|
|
||||||
if url not in existing_urls:
|
|
||||||
unique_channels.append(channel)
|
|
||||||
|
|
||||||
self.logger.info(f"Removed {len(new_channels) - len(unique_channels)} duplicate channels")
|
|
||||||
return unique_channels
|
|
||||||
|
|
||||||
def append_to_bulk_import(self, channels: List[str]):
|
|
||||||
"""Append discovered channels to bulk_import.m3u"""
|
|
||||||
if not channels:
|
|
||||||
self.logger.info("No new channels to add")
|
|
||||||
return
|
|
||||||
|
|
||||||
# Read existing bulk_import content
|
|
||||||
existing_content = ""
|
|
||||||
if os.path.exists('bulk_import.m3u'):
|
|
||||||
with open('bulk_import.m3u', 'r', encoding='utf-8') as f:
|
|
||||||
existing_content = f.read().strip()
|
|
||||||
|
|
||||||
# If file is empty or only has header, start fresh
|
|
||||||
if not existing_content or existing_content == '#EXTM3U':
|
|
||||||
existing_content = '#EXTM3U'
|
|
||||||
|
|
||||||
# Append new channels
|
|
||||||
with open('bulk_import.m3u', 'w', encoding='utf-8') as f:
|
|
||||||
f.write(existing_content)
|
|
||||||
if not existing_content.endswith('\n'):
|
|
||||||
f.write('\n')
|
|
||||||
f.write('\n'.join(channels))
|
|
||||||
f.write('\n')
|
|
||||||
|
|
||||||
self.logger.info(f"Added {len(channels)} new channels to bulk_import.m3u")
|
|
||||||
|
|
||||||
def generate_discovery_report(self):
|
|
||||||
"""Generate discovery session report"""
|
|
||||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
||||||
report_path = f"reports/daily/discovery_report_{timestamp}.md"
|
|
||||||
|
|
||||||
os.makedirs('reports/daily', exist_ok=True)
|
|
||||||
|
|
||||||
total_discovered = sum(self.source_stats.values())
|
|
||||||
|
|
||||||
with open(report_path, 'w', encoding='utf-8') as f:
|
|
||||||
f.write(f"# Channel Discovery Report\n")
|
|
||||||
f.write(f"**Generated:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n\n")
|
|
||||||
f.write(f"## Summary\n")
|
|
||||||
f.write(f"- **Total Channels Discovered:** {total_discovered}\n")
|
|
||||||
f.write(f"- **Sources Checked:** {len(self.config['sources'])}\n")
|
|
||||||
f.write(f"- **Active Sources:** {len(self.source_stats)}\n\n")
|
|
||||||
f.write(f"## Source Breakdown\n")
|
|
||||||
|
|
||||||
for source_name, count in self.source_stats.items():
|
|
||||||
f.write(f"- **{source_name}:** {count} channels\n")
|
|
||||||
|
|
||||||
f.write(f"\n## Configuration\n")
|
|
||||||
f.write(f"- **Max per source:** {self.config['filters']['max_channels_per_source']}\n")
|
|
||||||
f.write(f"- **Adult filter:** {'Enabled' if self.config['filters']['exclude_adult'] else 'Disabled'}\n")
|
|
||||||
f.write(f"- **Quality filter:** {self.config['filters']['min_quality']}\n")
|
|
||||||
f.write(f"\n---\n*Auto-generated by Source Scraper*\n")
|
|
||||||
|
|
||||||
self.logger.info(f"Discovery report saved: {report_path}")
|
|
||||||
|
|
||||||
def run_discovery(self):
|
|
||||||
"""Main discovery process"""
|
|
||||||
if not self.config['enabled']:
|
|
||||||
self.logger.info("Discovery is disabled in configuration")
|
|
||||||
return
|
|
||||||
|
|
||||||
self.logger.info("=== Starting Channel Discovery ===")
|
|
||||||
|
|
||||||
all_discovered = []
|
|
||||||
|
|
||||||
for source in self.config['sources']:
|
|
||||||
try:
|
|
||||||
if source['type'] == 'm3u_playlist':
|
|
||||||
channels = self.discover_from_m3u_url(source)
|
|
||||||
elif source['type'] == 'github_directory':
|
|
||||||
channels = self.discover_from_github_directory(source)
|
|
||||||
else:
|
|
||||||
self.logger.warning(f"Unknown source type: {source['type']}")
|
|
||||||
continue
|
|
||||||
|
|
||||||
all_discovered.extend(channels)
|
|
||||||
|
|
||||||
# Rate limiting
|
|
||||||
import time
|
|
||||||
time.sleep(self.config['rate_limiting']['delay_between_requests'])
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
self.logger.error(f"Error processing source {source['name']}: {e}")
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Apply filters
|
|
||||||
filtered_channels = self.filter_channels(all_discovered)
|
|
||||||
unique_channels = self.deduplicate_with_existing(filtered_channels)
|
|
||||||
|
|
||||||
# Add to bulk import
|
|
||||||
self.append_to_bulk_import(unique_channels)
|
|
||||||
|
|
||||||
# Generate report
|
|
||||||
self.generate_discovery_report()
|
|
||||||
|
|
||||||
self.logger.info(f"=== Discovery Complete: {len(unique_channels)} new channels added ===")
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
scraper = SourceScraper()
|
|
||||||
scraper.run_discovery()
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue