my-private-iptv-m3u/setup_clean_repo.py

552 lines
No EOL
16 KiB
Python
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""
Complete IPTV Repository Setup & Cleanup
This script will set up all the cleanup automation and organize your repository
"""
import os
import shutil
import subprocess
from pathlib import Path
from datetime import datetime
def run_command(cmd, description):
"""Run a command and show the result."""
print(f"🔧 {description}...")
try:
result = subprocess.run(cmd, shell=True, capture_output=True, text=True)
if result.returncode == 0:
print(f" ✅ Success")
return True
else:
print(f" ⚠️ Warning: {result.stderr.strip()}")
return False
except Exception as e:
print(f" ❌ Error: {e}")
return False
def create_file(file_path, content, description):
"""Create a file with given content."""
print(f"📝 Creating {description}...")
try:
file_path.parent.mkdir(parents=True, exist_ok=True)
with open(file_path, 'w', encoding='utf-8') as f:
f.write(content)
print(f" ✅ Created: {file_path}")
return True
except Exception as e:
print(f" ❌ Error creating {file_path}: {e}")
return False
def main():
"""Set up complete repository cleanup and automation."""
print("🎯 IPTV Repository Complete Setup")
print("=" * 50)
print("This will set up automated cleanup and organization for your repository")
print()
root_path = Path.cwd()
# Check if we're in the right directory
if not (root_path / 'scripts').exists():
print("❌ Error: Please run this script from your repository root directory")
print(" (The directory should contain a 'scripts' folder)")
return
print(f"📁 Working in: {root_path}")
print()
# Step 1: Create enhanced .gitignore
gitignore_content = """# IPTV Playlist Generator - Enhanced .gitignore
# ===== PYTHON =====
__pycache__/
*.py[cod]
*$py.class
*.so
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
# ===== LOGS & TEMPORARY FILES =====
*.log
*.tmp
*_temp*
*.backup.*
temp_*
.temp/
# Specific log patterns
playlist_update.log
import_*.log
health_check_*.log
# ===== BACKUP FILES =====
# Keep backups directory but ignore temporary backups
backups/temp_*
backups/*.tmp
*.backup
*~
# ===== DEVELOPMENT & TESTING =====
.pytest_cache/
.coverage
htmlcov/
.tox/
.nox/
.cache
nosetests.xml
coverage.xml
*.cover
.hypothesis/
# ===== IDE & EDITOR FILES =====
.vscode/
.idea/
*.swp
*.swo
*~
.DS_Store
Thumbs.db
# ===== ENVIRONMENT & CONFIG =====
.env
.env.local
.env.*.local
venv/
env/
ENV/
# ===== IPTV SPECIFIC =====
# Temporary import files (keep the main one)
bulk_import_temp.m3u
import_temp_*.m3u
*_processing.m3u
# Large test files
test_large_*.m3u
sample_*.m3u
# Generated temporary playlists
playlist_temp.m3u
temp_playlist_*.m3u
# ===== DOCUMENTATION BUILDS =====
docs/_build/
site/
# ===== ARCHIVE & COMPRESSED =====
*.tar.gz
*.zip
*.rar
logs_archive_*.tar.gz
# ===== SYSTEM FILES =====
.DS_Store
.DS_Store?
._*
.Spotlight-V100
.Trashes
ehthumbs.db
Thumbs.db"""
create_file(root_path / '.gitignore', gitignore_content, "Enhanced .gitignore")
# Step 2: Create repository health monitor
repo_health_content = '''#!/usr/bin/env python3
"""
Repository Health Monitor - Simplified for immediate use
"""
import os
import shutil
import logging
from datetime import datetime, timedelta
from pathlib import Path
import gzip
class SimpleRepoCleanup:
"""Simplified repository cleanup for immediate use."""
def __init__(self):
self.root_path = Path.cwd()
self.cleaned_items = []
def run_cleanup(self):
"""Run complete cleanup."""
print("🧹 Starting repository cleanup...")
# 1. Remove Python cache
self._clean_python_cache()
# 2. Remove temporary files
self._clean_temp_files()
# 3. Organize log files
self._organize_logs()
# 4. Compress old backups
self._compress_backups()
# 5. Ensure directory structure
self._ensure_directories()
# 6. Clean import file
self._clean_import_file()
print(f"\\n✅ Cleanup complete! {len(self.cleaned_items)} items processed")
return self.cleaned_items
def _clean_python_cache(self):
"""Remove Python cache files."""
print(" 🐍 Cleaning Python cache...")
# Remove __pycache__ directories
for cache_dir in self.root_path.rglob('__pycache__'):
if cache_dir.is_dir():
try:
shutil.rmtree(cache_dir)
self.cleaned_items.append(f"Removed cache: {cache_dir.relative_to(self.root_path)}")
except Exception:
pass
# Remove .pyc and .pyo files
for pyc_file in list(self.root_path.rglob('*.pyc')) + list(self.root_path.rglob('*.pyo')):
try:
pyc_file.unlink()
self.cleaned_items.append(f"Removed: {pyc_file.relative_to(self.root_path)}")
except Exception:
pass
def _clean_temp_files(self):
"""Remove temporary files."""
print(" 🗑️ Cleaning temporary files...")
patterns = ['*_temp*', '*.tmp', '*~', '*.swp', '*.swo']
for pattern in patterns:
for temp_file in self.root_path.rglob(pattern):
if temp_file.is_file() and '.git' not in str(temp_file):
try:
temp_file.unlink()
self.cleaned_items.append(f"Removed temp: {temp_file.relative_to(self.root_path)}")
except Exception:
pass
def _organize_logs(self):
"""Organize log files."""
print(" 📋 Organizing log files...")
logs_dir = self.root_path / 'reports' / 'logs'
logs_dir.mkdir(parents=True, exist_ok=True)
for log_file in self.root_path.glob('*.log'):
try:
new_location = logs_dir / log_file.name
shutil.move(str(log_file), str(new_location))
self.cleaned_items.append(f"Moved log: {log_file.name} → reports/logs/")
except Exception:
pass
def _compress_backups(self):
"""Compress old backup files."""
print(" 🗜️ Compressing old backups...")
backup_dir = self.root_path / 'backups'
if backup_dir.exists():
cutoff_date = datetime.now() - timedelta(days=7)
for backup_file in backup_dir.glob('*.txt'):
try:
file_date = datetime.fromtimestamp(backup_file.stat().st_mtime)
if file_date < cutoff_date:
# Compress with gzip
with open(backup_file, 'rb') as f_in:
with gzip.open(f"{backup_file}.gz", 'wb') as f_out:
shutil.copyfileobj(f_in, f_out)
backup_file.unlink()
self.cleaned_items.append(f"Compressed: {backup_file.name}")
except Exception:
pass
def _ensure_directories(self):
"""Ensure proper directory structure."""
print(" 📁 Ensuring directory structure...")
directories = [
'config',
'backups',
'reports/logs',
'reports/archive',
'templates'
]
for directory in directories:
(self.root_path / directory).mkdir(parents=True, exist_ok=True)
def _clean_import_file(self):
"""Clean the import file if needed."""
print(" 📥 Checking import file...")
import_file = self.root_path / 'bulk_import.m3u'
if import_file.exists():
try:
with open(import_file, 'r', encoding='utf-8') as f:
content = f.read().strip()
lines = content.split('\\n')
if len(lines) > 2 or any('#EXTINF' in line for line in lines):
with open(import_file, 'w', encoding='utf-8') as f:
f.write('#EXTM3U\\n')
self.cleaned_items.append("Cleared bulk_import.m3u (ready for next import)")
except Exception:
pass
def main():
"""Run the cleanup."""
cleanup = SimpleRepoCleanup()
cleanup.run_cleanup()
if __name__ == "__main__":
main()
'''
create_file(root_path / 'scripts' / 'quick_cleanup.py', repo_health_content, "Quick cleanup script")
# Step 3: Create the enhanced workflow
workflow_content = """name: Generate M3U Playlist with Auto-Cleanup
on:
push:
branches:
- main
workflow_dispatch:
jobs:
build-and-cleanup:
runs-on: ubuntu-22.04
steps:
- name: Checkout Repository
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.11'
- name: Configure Git
run: |
git config --local user.email "actions@forgejo.plainrock127.xyz"
git config --local user.name "IPTV Playlist Bot"
- name: Auto-Cleanup Repository
run: |
echo "=== Auto-Cleanup Phase ==="
# Remove Python cache thoroughly
find . -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true
find . -name "*.pyc" -delete 2>/dev/null || true
find . -name "*.pyo" -delete 2>/dev/null || true
# Remove temporary files
find . -name "*_temp*" -type f -delete 2>/dev/null || true
find . -name "*.tmp" -delete 2>/dev/null || true
find . -name "*~" -delete 2>/dev/null || true
# Clean backup files older than 30 days
find backups -name "*.txt" -type f -mtime +30 -delete 2>/dev/null || true
# Organize log files
mkdir -p reports/logs reports/archive
find . -maxdepth 1 -name "*.log" -exec mv {} reports/logs/ \\; 2>/dev/null || true
# Compress old backups (older than 7 days)
find backups -name "*.txt" -type f -mtime +7 -exec gzip {} \\; 2>/dev/null || true
echo "✅ Cleanup completed"
- name: Setup Directories
run: |
echo "=== Directory Setup ==="
mkdir -p config backups reports/logs reports/archive templates
# Create scripts/__init__.py if missing
if [ ! -f scripts/__init__.py ]; then
echo '# Scripts package' > scripts/__init__.py
fi
- name: Run Playlist Generation
run: |
echo "=== Playlist Generation ==="
python scripts/generate_playlist.py
- name: Post-Generation Analysis
run: |
echo "=== Results Analysis ==="
if [ -f playlist.m3u ]; then
CHANNEL_COUNT=$(grep -c "^#EXTINF" playlist.m3u 2>/dev/null || echo "0")
FILE_SIZE=$(du -h playlist.m3u | cut -f1)
echo "✅ playlist.m3u: $CHANNEL_COUNT channels ($FILE_SIZE)"
else
echo "❌ playlist.m3u not generated"
fi
- name: Final Cleanup & Organization
run: |
echo "=== Final Organization ==="
# Ensure bulk_import.m3u is clean
if [ -f bulk_import.m3u ]; then
LINE_COUNT=$(wc -l < bulk_import.m3u)
if [ "$LINE_COUNT" -gt 2 ]; then
echo '#EXTM3U' > bulk_import.m3u
echo '' >> bulk_import.m3u
echo "🧹 Cleaned bulk_import.m3u"
fi
fi
- name: Commit Changes
run: |
echo "=== Committing Changes ==="
# Add specific files only
git add bulk_import.m3u || true
git add channels.txt || true
git add playlist.m3u || true
git add scripts/ || true
git add config/ || true
git add reports/ || true
git add backups/*.gz || true
git add templates/ || true
git add .forgejo/ || true
git add README.md || true
git add .gitignore || true
if ! git diff --staged --quiet; then
CHANNEL_COUNT="0"
if [ -f playlist.m3u ]; then
CHANNEL_COUNT=$(grep -c "^#EXTINF" playlist.m3u 2>/dev/null || echo "0")
fi
git commit -m "📺 Updated playlist: $CHANNEL_COUNT channels ($(date '+%Y-%m-%d %H:%M')) - Auto-cleaned"
git push
echo "✅ Repository updated and cleaned"
else
echo " No changes to commit"
fi"""
workflow_dir = root_path / '.forgejo' / 'workflows'
create_file(workflow_dir / 'generate-m3u.yml', workflow_content, "Enhanced Forgejo workflow")
# Step 4: Run immediate cleanup
print("\n" + "=" * 50)
print("🧹 IMMEDIATE CLEANUP")
print("=" * 50)
# Import and run the cleanup
import sys
sys.path.insert(0, str(root_path / 'scripts'))
try:
exec(repo_health_content)
cleanup = SimpleRepoCleanup()
cleaned_items = cleanup.run_cleanup()
except Exception as e:
print(f"Running manual cleanup instead: {e}")
# Manual cleanup
print("🧹 Running manual cleanup...")
cleaned_items = []
# Remove Python cache
for cache_dir in root_path.rglob('__pycache__'):
if cache_dir.is_dir():
try:
shutil.rmtree(cache_dir)
cleaned_items.append(f"Removed: {cache_dir.relative_to(root_path)}")
except:
pass
# Remove temp files
patterns = ['*.pyc', '*.pyo', '*_temp*', '*.tmp', '*~']
for pattern in patterns:
for file_path in root_path.rglob(pattern):
if file_path.is_file() and '.git' not in str(file_path):
try:
file_path.unlink()
cleaned_items.append(f"Removed: {file_path.relative_to(root_path)}")
except:
pass
# Organize logs
logs_dir = root_path / 'reports' / 'logs'
logs_dir.mkdir(parents=True, exist_ok=True)
for log_file in root_path.glob('*.log'):
try:
shutil.move(str(log_file), str(logs_dir / log_file.name))
cleaned_items.append(f"Moved: {log_file.name} → reports/logs/")
except:
pass
# Step 5: Setup git hooks (optional)
print("\n" + "=" * 50)
print("🔧 OPTIONAL GIT SETUP")
print("=" * 50)
# Check if this is a git repository
if (root_path / '.git').exists():
print("📝 Setting up git configuration...")
# Add files to git
run_command("git add .gitignore", "Adding .gitignore")
run_command("git add scripts/quick_cleanup.py", "Adding cleanup script")
run_command("git add .forgejo/workflows/generate-m3u.yml", "Adding enhanced workflow")
print(" ✅ Files staged for commit")
else:
print(" Not a git repository, skipping git setup")
# Step 6: Final summary
print("\n" + "=" * 50)
print("✅ SETUP COMPLETE!")
print("=" * 50)
print(f"📊 Summary:")
print(f" - Cleaned {len(cleaned_items)} items")
print(f" - Created enhanced .gitignore")
print(f" - Added quick cleanup script")
print(f" - Updated Forgejo workflow")
print(f" - Organized directory structure")
print(f"\n🎯 What's New:")
print(f" 📁 reports/logs/ - All logs organized here")
print(f" 🧹 scripts/quick_cleanup.py - Manual cleanup tool")
print(f" 🔄 Enhanced workflow - Automatic cleanup on push")
print(f" 🚫 .gitignore - Prevents future clutter")
print(f"\n🚀 Next Steps:")
print(f" 1. Commit these changes: git commit -m 'Setup automated cleanup'")
print(f" 2. Push to trigger workflow: git push")
print(f" 3. For manual cleanup: python scripts/quick_cleanup.py")
print(f" 4. Your repository will now stay clean automatically!")
print(f"\n🎉 Your IPTV repository is now clean and organized!")
if __name__ == "__main__":
main()