552 lines
No EOL
16 KiB
Python
552 lines
No EOL
16 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
Complete IPTV Repository Setup & Cleanup
|
||
This script will set up all the cleanup automation and organize your repository
|
||
"""
|
||
|
||
import os
|
||
import shutil
|
||
import subprocess
|
||
from pathlib import Path
|
||
from datetime import datetime
|
||
|
||
def run_command(cmd, description):
|
||
"""Run a command and show the result."""
|
||
print(f"🔧 {description}...")
|
||
try:
|
||
result = subprocess.run(cmd, shell=True, capture_output=True, text=True)
|
||
if result.returncode == 0:
|
||
print(f" ✅ Success")
|
||
return True
|
||
else:
|
||
print(f" ⚠️ Warning: {result.stderr.strip()}")
|
||
return False
|
||
except Exception as e:
|
||
print(f" ❌ Error: {e}")
|
||
return False
|
||
|
||
def create_file(file_path, content, description):
|
||
"""Create a file with given content."""
|
||
print(f"📝 Creating {description}...")
|
||
try:
|
||
file_path.parent.mkdir(parents=True, exist_ok=True)
|
||
with open(file_path, 'w', encoding='utf-8') as f:
|
||
f.write(content)
|
||
print(f" ✅ Created: {file_path}")
|
||
return True
|
||
except Exception as e:
|
||
print(f" ❌ Error creating {file_path}: {e}")
|
||
return False
|
||
|
||
def main():
|
||
"""Set up complete repository cleanup and automation."""
|
||
print("🎯 IPTV Repository Complete Setup")
|
||
print("=" * 50)
|
||
print("This will set up automated cleanup and organization for your repository")
|
||
print()
|
||
|
||
root_path = Path.cwd()
|
||
|
||
# Check if we're in the right directory
|
||
if not (root_path / 'scripts').exists():
|
||
print("❌ Error: Please run this script from your repository root directory")
|
||
print(" (The directory should contain a 'scripts' folder)")
|
||
return
|
||
|
||
print(f"📁 Working in: {root_path}")
|
||
print()
|
||
|
||
# Step 1: Create enhanced .gitignore
|
||
gitignore_content = """# IPTV Playlist Generator - Enhanced .gitignore
|
||
|
||
# ===== PYTHON =====
|
||
__pycache__/
|
||
*.py[cod]
|
||
*$py.class
|
||
*.so
|
||
.Python
|
||
build/
|
||
develop-eggs/
|
||
dist/
|
||
downloads/
|
||
eggs/
|
||
.eggs/
|
||
lib/
|
||
lib64/
|
||
parts/
|
||
sdist/
|
||
var/
|
||
wheels/
|
||
*.egg-info/
|
||
.installed.cfg
|
||
*.egg
|
||
|
||
# ===== LOGS & TEMPORARY FILES =====
|
||
*.log
|
||
*.tmp
|
||
*_temp*
|
||
*.backup.*
|
||
temp_*
|
||
.temp/
|
||
|
||
# Specific log patterns
|
||
playlist_update.log
|
||
import_*.log
|
||
health_check_*.log
|
||
|
||
# ===== BACKUP FILES =====
|
||
# Keep backups directory but ignore temporary backups
|
||
backups/temp_*
|
||
backups/*.tmp
|
||
*.backup
|
||
*~
|
||
|
||
# ===== DEVELOPMENT & TESTING =====
|
||
.pytest_cache/
|
||
.coverage
|
||
htmlcov/
|
||
.tox/
|
||
.nox/
|
||
.cache
|
||
nosetests.xml
|
||
coverage.xml
|
||
*.cover
|
||
.hypothesis/
|
||
|
||
# ===== IDE & EDITOR FILES =====
|
||
.vscode/
|
||
.idea/
|
||
*.swp
|
||
*.swo
|
||
*~
|
||
.DS_Store
|
||
Thumbs.db
|
||
|
||
# ===== ENVIRONMENT & CONFIG =====
|
||
.env
|
||
.env.local
|
||
.env.*.local
|
||
venv/
|
||
env/
|
||
ENV/
|
||
|
||
# ===== IPTV SPECIFIC =====
|
||
# Temporary import files (keep the main one)
|
||
bulk_import_temp.m3u
|
||
import_temp_*.m3u
|
||
*_processing.m3u
|
||
|
||
# Large test files
|
||
test_large_*.m3u
|
||
sample_*.m3u
|
||
|
||
# Generated temporary playlists
|
||
playlist_temp.m3u
|
||
temp_playlist_*.m3u
|
||
|
||
# ===== DOCUMENTATION BUILDS =====
|
||
docs/_build/
|
||
site/
|
||
|
||
# ===== ARCHIVE & COMPRESSED =====
|
||
*.tar.gz
|
||
*.zip
|
||
*.rar
|
||
logs_archive_*.tar.gz
|
||
|
||
# ===== SYSTEM FILES =====
|
||
.DS_Store
|
||
.DS_Store?
|
||
._*
|
||
.Spotlight-V100
|
||
.Trashes
|
||
ehthumbs.db
|
||
Thumbs.db"""
|
||
|
||
create_file(root_path / '.gitignore', gitignore_content, "Enhanced .gitignore")
|
||
|
||
# Step 2: Create repository health monitor
|
||
repo_health_content = '''#!/usr/bin/env python3
|
||
"""
|
||
Repository Health Monitor - Simplified for immediate use
|
||
"""
|
||
|
||
import os
|
||
import shutil
|
||
import logging
|
||
from datetime import datetime, timedelta
|
||
from pathlib import Path
|
||
import gzip
|
||
|
||
class SimpleRepoCleanup:
|
||
"""Simplified repository cleanup for immediate use."""
|
||
|
||
def __init__(self):
|
||
self.root_path = Path.cwd()
|
||
self.cleaned_items = []
|
||
|
||
def run_cleanup(self):
|
||
"""Run complete cleanup."""
|
||
print("🧹 Starting repository cleanup...")
|
||
|
||
# 1. Remove Python cache
|
||
self._clean_python_cache()
|
||
|
||
# 2. Remove temporary files
|
||
self._clean_temp_files()
|
||
|
||
# 3. Organize log files
|
||
self._organize_logs()
|
||
|
||
# 4. Compress old backups
|
||
self._compress_backups()
|
||
|
||
# 5. Ensure directory structure
|
||
self._ensure_directories()
|
||
|
||
# 6. Clean import file
|
||
self._clean_import_file()
|
||
|
||
print(f"\\n✅ Cleanup complete! {len(self.cleaned_items)} items processed")
|
||
|
||
return self.cleaned_items
|
||
|
||
def _clean_python_cache(self):
|
||
"""Remove Python cache files."""
|
||
print(" 🐍 Cleaning Python cache...")
|
||
|
||
# Remove __pycache__ directories
|
||
for cache_dir in self.root_path.rglob('__pycache__'):
|
||
if cache_dir.is_dir():
|
||
try:
|
||
shutil.rmtree(cache_dir)
|
||
self.cleaned_items.append(f"Removed cache: {cache_dir.relative_to(self.root_path)}")
|
||
except Exception:
|
||
pass
|
||
|
||
# Remove .pyc and .pyo files
|
||
for pyc_file in list(self.root_path.rglob('*.pyc')) + list(self.root_path.rglob('*.pyo')):
|
||
try:
|
||
pyc_file.unlink()
|
||
self.cleaned_items.append(f"Removed: {pyc_file.relative_to(self.root_path)}")
|
||
except Exception:
|
||
pass
|
||
|
||
def _clean_temp_files(self):
|
||
"""Remove temporary files."""
|
||
print(" 🗑️ Cleaning temporary files...")
|
||
|
||
patterns = ['*_temp*', '*.tmp', '*~', '*.swp', '*.swo']
|
||
|
||
for pattern in patterns:
|
||
for temp_file in self.root_path.rglob(pattern):
|
||
if temp_file.is_file() and '.git' not in str(temp_file):
|
||
try:
|
||
temp_file.unlink()
|
||
self.cleaned_items.append(f"Removed temp: {temp_file.relative_to(self.root_path)}")
|
||
except Exception:
|
||
pass
|
||
|
||
def _organize_logs(self):
|
||
"""Organize log files."""
|
||
print(" 📋 Organizing log files...")
|
||
|
||
logs_dir = self.root_path / 'reports' / 'logs'
|
||
logs_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
for log_file in self.root_path.glob('*.log'):
|
||
try:
|
||
new_location = logs_dir / log_file.name
|
||
shutil.move(str(log_file), str(new_location))
|
||
self.cleaned_items.append(f"Moved log: {log_file.name} → reports/logs/")
|
||
except Exception:
|
||
pass
|
||
|
||
def _compress_backups(self):
|
||
"""Compress old backup files."""
|
||
print(" 🗜️ Compressing old backups...")
|
||
|
||
backup_dir = self.root_path / 'backups'
|
||
if backup_dir.exists():
|
||
cutoff_date = datetime.now() - timedelta(days=7)
|
||
|
||
for backup_file in backup_dir.glob('*.txt'):
|
||
try:
|
||
file_date = datetime.fromtimestamp(backup_file.stat().st_mtime)
|
||
if file_date < cutoff_date:
|
||
# Compress with gzip
|
||
with open(backup_file, 'rb') as f_in:
|
||
with gzip.open(f"{backup_file}.gz", 'wb') as f_out:
|
||
shutil.copyfileobj(f_in, f_out)
|
||
backup_file.unlink()
|
||
self.cleaned_items.append(f"Compressed: {backup_file.name}")
|
||
except Exception:
|
||
pass
|
||
|
||
def _ensure_directories(self):
|
||
"""Ensure proper directory structure."""
|
||
print(" 📁 Ensuring directory structure...")
|
||
|
||
directories = [
|
||
'config',
|
||
'backups',
|
||
'reports/logs',
|
||
'reports/archive',
|
||
'templates'
|
||
]
|
||
|
||
for directory in directories:
|
||
(self.root_path / directory).mkdir(parents=True, exist_ok=True)
|
||
|
||
def _clean_import_file(self):
|
||
"""Clean the import file if needed."""
|
||
print(" 📥 Checking import file...")
|
||
|
||
import_file = self.root_path / 'bulk_import.m3u'
|
||
if import_file.exists():
|
||
try:
|
||
with open(import_file, 'r', encoding='utf-8') as f:
|
||
content = f.read().strip()
|
||
|
||
lines = content.split('\\n')
|
||
if len(lines) > 2 or any('#EXTINF' in line for line in lines):
|
||
with open(import_file, 'w', encoding='utf-8') as f:
|
||
f.write('#EXTM3U\\n')
|
||
self.cleaned_items.append("Cleared bulk_import.m3u (ready for next import)")
|
||
except Exception:
|
||
pass
|
||
|
||
def main():
|
||
"""Run the cleanup."""
|
||
cleanup = SimpleRepoCleanup()
|
||
cleanup.run_cleanup()
|
||
|
||
if __name__ == "__main__":
|
||
main()
|
||
'''
|
||
|
||
create_file(root_path / 'scripts' / 'quick_cleanup.py', repo_health_content, "Quick cleanup script")
|
||
|
||
# Step 3: Create the enhanced workflow
|
||
workflow_content = """name: Generate M3U Playlist with Auto-Cleanup
|
||
|
||
on:
|
||
push:
|
||
branches:
|
||
- main
|
||
workflow_dispatch:
|
||
|
||
jobs:
|
||
build-and-cleanup:
|
||
runs-on: ubuntu-22.04
|
||
steps:
|
||
- name: Checkout Repository
|
||
uses: actions/checkout@v4
|
||
|
||
- name: Set up Python
|
||
uses: actions/setup-python@v5
|
||
with:
|
||
python-version: '3.11'
|
||
|
||
- name: Configure Git
|
||
run: |
|
||
git config --local user.email "actions@forgejo.plainrock127.xyz"
|
||
git config --local user.name "IPTV Playlist Bot"
|
||
|
||
- name: Auto-Cleanup Repository
|
||
run: |
|
||
echo "=== Auto-Cleanup Phase ==="
|
||
|
||
# Remove Python cache thoroughly
|
||
find . -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true
|
||
find . -name "*.pyc" -delete 2>/dev/null || true
|
||
find . -name "*.pyo" -delete 2>/dev/null || true
|
||
|
||
# Remove temporary files
|
||
find . -name "*_temp*" -type f -delete 2>/dev/null || true
|
||
find . -name "*.tmp" -delete 2>/dev/null || true
|
||
find . -name "*~" -delete 2>/dev/null || true
|
||
|
||
# Clean backup files older than 30 days
|
||
find backups -name "*.txt" -type f -mtime +30 -delete 2>/dev/null || true
|
||
|
||
# Organize log files
|
||
mkdir -p reports/logs reports/archive
|
||
find . -maxdepth 1 -name "*.log" -exec mv {} reports/logs/ \\; 2>/dev/null || true
|
||
|
||
# Compress old backups (older than 7 days)
|
||
find backups -name "*.txt" -type f -mtime +7 -exec gzip {} \\; 2>/dev/null || true
|
||
|
||
echo "✅ Cleanup completed"
|
||
|
||
- name: Setup Directories
|
||
run: |
|
||
echo "=== Directory Setup ==="
|
||
mkdir -p config backups reports/logs reports/archive templates
|
||
|
||
# Create scripts/__init__.py if missing
|
||
if [ ! -f scripts/__init__.py ]; then
|
||
echo '# Scripts package' > scripts/__init__.py
|
||
fi
|
||
|
||
- name: Run Playlist Generation
|
||
run: |
|
||
echo "=== Playlist Generation ==="
|
||
python scripts/generate_playlist.py
|
||
|
||
- name: Post-Generation Analysis
|
||
run: |
|
||
echo "=== Results Analysis ==="
|
||
|
||
if [ -f playlist.m3u ]; then
|
||
CHANNEL_COUNT=$(grep -c "^#EXTINF" playlist.m3u 2>/dev/null || echo "0")
|
||
FILE_SIZE=$(du -h playlist.m3u | cut -f1)
|
||
echo "✅ playlist.m3u: $CHANNEL_COUNT channels ($FILE_SIZE)"
|
||
else
|
||
echo "❌ playlist.m3u not generated"
|
||
fi
|
||
|
||
- name: Final Cleanup & Organization
|
||
run: |
|
||
echo "=== Final Organization ==="
|
||
|
||
# Ensure bulk_import.m3u is clean
|
||
if [ -f bulk_import.m3u ]; then
|
||
LINE_COUNT=$(wc -l < bulk_import.m3u)
|
||
if [ "$LINE_COUNT" -gt 2 ]; then
|
||
echo '#EXTM3U' > bulk_import.m3u
|
||
echo '' >> bulk_import.m3u
|
||
echo "🧹 Cleaned bulk_import.m3u"
|
||
fi
|
||
fi
|
||
|
||
- name: Commit Changes
|
||
run: |
|
||
echo "=== Committing Changes ==="
|
||
|
||
# Add specific files only
|
||
git add bulk_import.m3u || true
|
||
git add channels.txt || true
|
||
git add playlist.m3u || true
|
||
git add scripts/ || true
|
||
git add config/ || true
|
||
git add reports/ || true
|
||
git add backups/*.gz || true
|
||
git add templates/ || true
|
||
git add .forgejo/ || true
|
||
git add README.md || true
|
||
git add .gitignore || true
|
||
|
||
if ! git diff --staged --quiet; then
|
||
CHANNEL_COUNT="0"
|
||
if [ -f playlist.m3u ]; then
|
||
CHANNEL_COUNT=$(grep -c "^#EXTINF" playlist.m3u 2>/dev/null || echo "0")
|
||
fi
|
||
|
||
git commit -m "📺 Updated playlist: $CHANNEL_COUNT channels ($(date '+%Y-%m-%d %H:%M')) - Auto-cleaned"
|
||
git push
|
||
echo "✅ Repository updated and cleaned"
|
||
else
|
||
echo "ℹ️ No changes to commit"
|
||
fi"""
|
||
|
||
workflow_dir = root_path / '.forgejo' / 'workflows'
|
||
create_file(workflow_dir / 'generate-m3u.yml', workflow_content, "Enhanced Forgejo workflow")
|
||
|
||
# Step 4: Run immediate cleanup
|
||
print("\n" + "=" * 50)
|
||
print("🧹 IMMEDIATE CLEANUP")
|
||
print("=" * 50)
|
||
|
||
# Import and run the cleanup
|
||
import sys
|
||
sys.path.insert(0, str(root_path / 'scripts'))
|
||
|
||
try:
|
||
exec(repo_health_content)
|
||
cleanup = SimpleRepoCleanup()
|
||
cleaned_items = cleanup.run_cleanup()
|
||
except Exception as e:
|
||
print(f"Running manual cleanup instead: {e}")
|
||
|
||
# Manual cleanup
|
||
print("🧹 Running manual cleanup...")
|
||
cleaned_items = []
|
||
|
||
# Remove Python cache
|
||
for cache_dir in root_path.rglob('__pycache__'):
|
||
if cache_dir.is_dir():
|
||
try:
|
||
shutil.rmtree(cache_dir)
|
||
cleaned_items.append(f"Removed: {cache_dir.relative_to(root_path)}")
|
||
except:
|
||
pass
|
||
|
||
# Remove temp files
|
||
patterns = ['*.pyc', '*.pyo', '*_temp*', '*.tmp', '*~']
|
||
for pattern in patterns:
|
||
for file_path in root_path.rglob(pattern):
|
||
if file_path.is_file() and '.git' not in str(file_path):
|
||
try:
|
||
file_path.unlink()
|
||
cleaned_items.append(f"Removed: {file_path.relative_to(root_path)}")
|
||
except:
|
||
pass
|
||
|
||
# Organize logs
|
||
logs_dir = root_path / 'reports' / 'logs'
|
||
logs_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
for log_file in root_path.glob('*.log'):
|
||
try:
|
||
shutil.move(str(log_file), str(logs_dir / log_file.name))
|
||
cleaned_items.append(f"Moved: {log_file.name} → reports/logs/")
|
||
except:
|
||
pass
|
||
|
||
# Step 5: Setup git hooks (optional)
|
||
print("\n" + "=" * 50)
|
||
print("🔧 OPTIONAL GIT SETUP")
|
||
print("=" * 50)
|
||
|
||
# Check if this is a git repository
|
||
if (root_path / '.git').exists():
|
||
print("📝 Setting up git configuration...")
|
||
|
||
# Add files to git
|
||
run_command("git add .gitignore", "Adding .gitignore")
|
||
run_command("git add scripts/quick_cleanup.py", "Adding cleanup script")
|
||
run_command("git add .forgejo/workflows/generate-m3u.yml", "Adding enhanced workflow")
|
||
|
||
print(" ✅ Files staged for commit")
|
||
else:
|
||
print(" ℹ️ Not a git repository, skipping git setup")
|
||
|
||
# Step 6: Final summary
|
||
print("\n" + "=" * 50)
|
||
print("✅ SETUP COMPLETE!")
|
||
print("=" * 50)
|
||
|
||
print(f"📊 Summary:")
|
||
print(f" - Cleaned {len(cleaned_items)} items")
|
||
print(f" - Created enhanced .gitignore")
|
||
print(f" - Added quick cleanup script")
|
||
print(f" - Updated Forgejo workflow")
|
||
print(f" - Organized directory structure")
|
||
|
||
print(f"\n🎯 What's New:")
|
||
print(f" 📁 reports/logs/ - All logs organized here")
|
||
print(f" 🧹 scripts/quick_cleanup.py - Manual cleanup tool")
|
||
print(f" 🔄 Enhanced workflow - Automatic cleanup on push")
|
||
print(f" 🚫 .gitignore - Prevents future clutter")
|
||
|
||
print(f"\n🚀 Next Steps:")
|
||
print(f" 1. Commit these changes: git commit -m 'Setup automated cleanup'")
|
||
print(f" 2. Push to trigger workflow: git push")
|
||
print(f" 3. For manual cleanup: python scripts/quick_cleanup.py")
|
||
print(f" 4. Your repository will now stay clean automatically!")
|
||
|
||
print(f"\n🎉 Your IPTV repository is now clean and organized!")
|
||
|
||
if __name__ == "__main__":
|
||
main() |