Add setup_clean_repo.py
This commit is contained in:
parent
805de97c7f
commit
d27c05b9d8
1 changed files with 552 additions and 0 deletions
552
setup_clean_repo.py
Normal file
552
setup_clean_repo.py
Normal file
|
|
@ -0,0 +1,552 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Complete IPTV Repository Setup & Cleanup
|
||||
This script will set up all the cleanup automation and organize your repository
|
||||
"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
|
||||
def run_command(cmd, description):
|
||||
"""Run a command and show the result."""
|
||||
print(f"🔧 {description}...")
|
||||
try:
|
||||
result = subprocess.run(cmd, shell=True, capture_output=True, text=True)
|
||||
if result.returncode == 0:
|
||||
print(f" ✅ Success")
|
||||
return True
|
||||
else:
|
||||
print(f" ⚠️ Warning: {result.stderr.strip()}")
|
||||
return False
|
||||
except Exception as e:
|
||||
print(f" ❌ Error: {e}")
|
||||
return False
|
||||
|
||||
def create_file(file_path, content, description):
|
||||
"""Create a file with given content."""
|
||||
print(f"📝 Creating {description}...")
|
||||
try:
|
||||
file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(file_path, 'w', encoding='utf-8') as f:
|
||||
f.write(content)
|
||||
print(f" ✅ Created: {file_path}")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f" ❌ Error creating {file_path}: {e}")
|
||||
return False
|
||||
|
||||
def main():
|
||||
"""Set up complete repository cleanup and automation."""
|
||||
print("🎯 IPTV Repository Complete Setup")
|
||||
print("=" * 50)
|
||||
print("This will set up automated cleanup and organization for your repository")
|
||||
print()
|
||||
|
||||
root_path = Path.cwd()
|
||||
|
||||
# Check if we're in the right directory
|
||||
if not (root_path / 'scripts').exists():
|
||||
print("❌ Error: Please run this script from your repository root directory")
|
||||
print(" (The directory should contain a 'scripts' folder)")
|
||||
return
|
||||
|
||||
print(f"📁 Working in: {root_path}")
|
||||
print()
|
||||
|
||||
# Step 1: Create enhanced .gitignore
|
||||
gitignore_content = """# IPTV Playlist Generator - Enhanced .gitignore
|
||||
|
||||
# ===== PYTHON =====
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.so
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
|
||||
# ===== LOGS & TEMPORARY FILES =====
|
||||
*.log
|
||||
*.tmp
|
||||
*_temp*
|
||||
*.backup.*
|
||||
temp_*
|
||||
.temp/
|
||||
|
||||
# Specific log patterns
|
||||
playlist_update.log
|
||||
import_*.log
|
||||
health_check_*.log
|
||||
|
||||
# ===== BACKUP FILES =====
|
||||
# Keep backups directory but ignore temporary backups
|
||||
backups/temp_*
|
||||
backups/*.tmp
|
||||
*.backup
|
||||
*~
|
||||
|
||||
# ===== DEVELOPMENT & TESTING =====
|
||||
.pytest_cache/
|
||||
.coverage
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
.hypothesis/
|
||||
|
||||
# ===== IDE & EDITOR FILES =====
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# ===== ENVIRONMENT & CONFIG =====
|
||||
.env
|
||||
.env.local
|
||||
.env.*.local
|
||||
venv/
|
||||
env/
|
||||
ENV/
|
||||
|
||||
# ===== IPTV SPECIFIC =====
|
||||
# Temporary import files (keep the main one)
|
||||
bulk_import_temp.m3u
|
||||
import_temp_*.m3u
|
||||
*_processing.m3u
|
||||
|
||||
# Large test files
|
||||
test_large_*.m3u
|
||||
sample_*.m3u
|
||||
|
||||
# Generated temporary playlists
|
||||
playlist_temp.m3u
|
||||
temp_playlist_*.m3u
|
||||
|
||||
# ===== DOCUMENTATION BUILDS =====
|
||||
docs/_build/
|
||||
site/
|
||||
|
||||
# ===== ARCHIVE & COMPRESSED =====
|
||||
*.tar.gz
|
||||
*.zip
|
||||
*.rar
|
||||
logs_archive_*.tar.gz
|
||||
|
||||
# ===== SYSTEM FILES =====
|
||||
.DS_Store
|
||||
.DS_Store?
|
||||
._*
|
||||
.Spotlight-V100
|
||||
.Trashes
|
||||
ehthumbs.db
|
||||
Thumbs.db"""
|
||||
|
||||
create_file(root_path / '.gitignore', gitignore_content, "Enhanced .gitignore")
|
||||
|
||||
# Step 2: Create repository health monitor
|
||||
repo_health_content = '''#!/usr/bin/env python3
|
||||
"""
|
||||
Repository Health Monitor - Simplified for immediate use
|
||||
"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
import gzip
|
||||
|
||||
class SimpleRepoCleanup:
|
||||
"""Simplified repository cleanup for immediate use."""
|
||||
|
||||
def __init__(self):
|
||||
self.root_path = Path.cwd()
|
||||
self.cleaned_items = []
|
||||
|
||||
def run_cleanup(self):
|
||||
"""Run complete cleanup."""
|
||||
print("🧹 Starting repository cleanup...")
|
||||
|
||||
# 1. Remove Python cache
|
||||
self._clean_python_cache()
|
||||
|
||||
# 2. Remove temporary files
|
||||
self._clean_temp_files()
|
||||
|
||||
# 3. Organize log files
|
||||
self._organize_logs()
|
||||
|
||||
# 4. Compress old backups
|
||||
self._compress_backups()
|
||||
|
||||
# 5. Ensure directory structure
|
||||
self._ensure_directories()
|
||||
|
||||
# 6. Clean import file
|
||||
self._clean_import_file()
|
||||
|
||||
print(f"\\n✅ Cleanup complete! {len(self.cleaned_items)} items processed")
|
||||
|
||||
return self.cleaned_items
|
||||
|
||||
def _clean_python_cache(self):
|
||||
"""Remove Python cache files."""
|
||||
print(" 🐍 Cleaning Python cache...")
|
||||
|
||||
# Remove __pycache__ directories
|
||||
for cache_dir in self.root_path.rglob('__pycache__'):
|
||||
if cache_dir.is_dir():
|
||||
try:
|
||||
shutil.rmtree(cache_dir)
|
||||
self.cleaned_items.append(f"Removed cache: {cache_dir.relative_to(self.root_path)}")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Remove .pyc and .pyo files
|
||||
for pyc_file in list(self.root_path.rglob('*.pyc')) + list(self.root_path.rglob('*.pyo')):
|
||||
try:
|
||||
pyc_file.unlink()
|
||||
self.cleaned_items.append(f"Removed: {pyc_file.relative_to(self.root_path)}")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _clean_temp_files(self):
|
||||
"""Remove temporary files."""
|
||||
print(" 🗑️ Cleaning temporary files...")
|
||||
|
||||
patterns = ['*_temp*', '*.tmp', '*~', '*.swp', '*.swo']
|
||||
|
||||
for pattern in patterns:
|
||||
for temp_file in self.root_path.rglob(pattern):
|
||||
if temp_file.is_file() and '.git' not in str(temp_file):
|
||||
try:
|
||||
temp_file.unlink()
|
||||
self.cleaned_items.append(f"Removed temp: {temp_file.relative_to(self.root_path)}")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _organize_logs(self):
|
||||
"""Organize log files."""
|
||||
print(" 📋 Organizing log files...")
|
||||
|
||||
logs_dir = self.root_path / 'reports' / 'logs'
|
||||
logs_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for log_file in self.root_path.glob('*.log'):
|
||||
try:
|
||||
new_location = logs_dir / log_file.name
|
||||
shutil.move(str(log_file), str(new_location))
|
||||
self.cleaned_items.append(f"Moved log: {log_file.name} → reports/logs/")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _compress_backups(self):
|
||||
"""Compress old backup files."""
|
||||
print(" 🗜️ Compressing old backups...")
|
||||
|
||||
backup_dir = self.root_path / 'backups'
|
||||
if backup_dir.exists():
|
||||
cutoff_date = datetime.now() - timedelta(days=7)
|
||||
|
||||
for backup_file in backup_dir.glob('*.txt'):
|
||||
try:
|
||||
file_date = datetime.fromtimestamp(backup_file.stat().st_mtime)
|
||||
if file_date < cutoff_date:
|
||||
# Compress with gzip
|
||||
with open(backup_file, 'rb') as f_in:
|
||||
with gzip.open(f"{backup_file}.gz", 'wb') as f_out:
|
||||
shutil.copyfileobj(f_in, f_out)
|
||||
backup_file.unlink()
|
||||
self.cleaned_items.append(f"Compressed: {backup_file.name}")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _ensure_directories(self):
|
||||
"""Ensure proper directory structure."""
|
||||
print(" 📁 Ensuring directory structure...")
|
||||
|
||||
directories = [
|
||||
'config',
|
||||
'backups',
|
||||
'reports/logs',
|
||||
'reports/archive',
|
||||
'templates'
|
||||
]
|
||||
|
||||
for directory in directories:
|
||||
(self.root_path / directory).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def _clean_import_file(self):
|
||||
"""Clean the import file if needed."""
|
||||
print(" 📥 Checking import file...")
|
||||
|
||||
import_file = self.root_path / 'bulk_import.m3u'
|
||||
if import_file.exists():
|
||||
try:
|
||||
with open(import_file, 'r', encoding='utf-8') as f:
|
||||
content = f.read().strip()
|
||||
|
||||
lines = content.split('\\n')
|
||||
if len(lines) > 2 or any('#EXTINF' in line for line in lines):
|
||||
with open(import_file, 'w', encoding='utf-8') as f:
|
||||
f.write('#EXTM3U\\n')
|
||||
self.cleaned_items.append("Cleared bulk_import.m3u (ready for next import)")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def main():
|
||||
"""Run the cleanup."""
|
||||
cleanup = SimpleRepoCleanup()
|
||||
cleanup.run_cleanup()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
'''
|
||||
|
||||
create_file(root_path / 'scripts' / 'quick_cleanup.py', repo_health_content, "Quick cleanup script")
|
||||
|
||||
# Step 3: Create the enhanced workflow
|
||||
workflow_content = """name: Generate M3U Playlist with Auto-Cleanup
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
build-and-cleanup:
|
||||
runs-on: ubuntu-22.04
|
||||
steps:
|
||||
- name: Checkout Repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
- name: Configure Git
|
||||
run: |
|
||||
git config --local user.email "actions@forgejo.plainrock127.xyz"
|
||||
git config --local user.name "IPTV Playlist Bot"
|
||||
|
||||
- name: Auto-Cleanup Repository
|
||||
run: |
|
||||
echo "=== Auto-Cleanup Phase ==="
|
||||
|
||||
# Remove Python cache thoroughly
|
||||
find . -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true
|
||||
find . -name "*.pyc" -delete 2>/dev/null || true
|
||||
find . -name "*.pyo" -delete 2>/dev/null || true
|
||||
|
||||
# Remove temporary files
|
||||
find . -name "*_temp*" -type f -delete 2>/dev/null || true
|
||||
find . -name "*.tmp" -delete 2>/dev/null || true
|
||||
find . -name "*~" -delete 2>/dev/null || true
|
||||
|
||||
# Clean backup files older than 30 days
|
||||
find backups -name "*.txt" -type f -mtime +30 -delete 2>/dev/null || true
|
||||
|
||||
# Organize log files
|
||||
mkdir -p reports/logs reports/archive
|
||||
find . -maxdepth 1 -name "*.log" -exec mv {} reports/logs/ \\; 2>/dev/null || true
|
||||
|
||||
# Compress old backups (older than 7 days)
|
||||
find backups -name "*.txt" -type f -mtime +7 -exec gzip {} \\; 2>/dev/null || true
|
||||
|
||||
echo "✅ Cleanup completed"
|
||||
|
||||
- name: Setup Directories
|
||||
run: |
|
||||
echo "=== Directory Setup ==="
|
||||
mkdir -p config backups reports/logs reports/archive templates
|
||||
|
||||
# Create scripts/__init__.py if missing
|
||||
if [ ! -f scripts/__init__.py ]; then
|
||||
echo '# Scripts package' > scripts/__init__.py
|
||||
fi
|
||||
|
||||
- name: Run Playlist Generation
|
||||
run: |
|
||||
echo "=== Playlist Generation ==="
|
||||
python scripts/generate_playlist.py
|
||||
|
||||
- name: Post-Generation Analysis
|
||||
run: |
|
||||
echo "=== Results Analysis ==="
|
||||
|
||||
if [ -f playlist.m3u ]; then
|
||||
CHANNEL_COUNT=$(grep -c "^#EXTINF" playlist.m3u 2>/dev/null || echo "0")
|
||||
FILE_SIZE=$(du -h playlist.m3u | cut -f1)
|
||||
echo "✅ playlist.m3u: $CHANNEL_COUNT channels ($FILE_SIZE)"
|
||||
else
|
||||
echo "❌ playlist.m3u not generated"
|
||||
fi
|
||||
|
||||
- name: Final Cleanup & Organization
|
||||
run: |
|
||||
echo "=== Final Organization ==="
|
||||
|
||||
# Ensure bulk_import.m3u is clean
|
||||
if [ -f bulk_import.m3u ]; then
|
||||
LINE_COUNT=$(wc -l < bulk_import.m3u)
|
||||
if [ "$LINE_COUNT" -gt 2 ]; then
|
||||
echo '#EXTM3U' > bulk_import.m3u
|
||||
echo '' >> bulk_import.m3u
|
||||
echo "🧹 Cleaned bulk_import.m3u"
|
||||
fi
|
||||
fi
|
||||
|
||||
- name: Commit Changes
|
||||
run: |
|
||||
echo "=== Committing Changes ==="
|
||||
|
||||
# Add specific files only
|
||||
git add bulk_import.m3u || true
|
||||
git add channels.txt || true
|
||||
git add playlist.m3u || true
|
||||
git add scripts/ || true
|
||||
git add config/ || true
|
||||
git add reports/ || true
|
||||
git add backups/*.gz || true
|
||||
git add templates/ || true
|
||||
git add .forgejo/ || true
|
||||
git add README.md || true
|
||||
git add .gitignore || true
|
||||
|
||||
if ! git diff --staged --quiet; then
|
||||
CHANNEL_COUNT="0"
|
||||
if [ -f playlist.m3u ]; then
|
||||
CHANNEL_COUNT=$(grep -c "^#EXTINF" playlist.m3u 2>/dev/null || echo "0")
|
||||
fi
|
||||
|
||||
git commit -m "📺 Updated playlist: $CHANNEL_COUNT channels ($(date '+%Y-%m-%d %H:%M')) - Auto-cleaned"
|
||||
git push
|
||||
echo "✅ Repository updated and cleaned"
|
||||
else
|
||||
echo "ℹ️ No changes to commit"
|
||||
fi"""
|
||||
|
||||
workflow_dir = root_path / '.forgejo' / 'workflows'
|
||||
create_file(workflow_dir / 'generate-m3u.yml', workflow_content, "Enhanced Forgejo workflow")
|
||||
|
||||
# Step 4: Run immediate cleanup
|
||||
print("\n" + "=" * 50)
|
||||
print("🧹 IMMEDIATE CLEANUP")
|
||||
print("=" * 50)
|
||||
|
||||
# Import and run the cleanup
|
||||
import sys
|
||||
sys.path.insert(0, str(root_path / 'scripts'))
|
||||
|
||||
try:
|
||||
exec(repo_health_content)
|
||||
cleanup = SimpleRepoCleanup()
|
||||
cleaned_items = cleanup.run_cleanup()
|
||||
except Exception as e:
|
||||
print(f"Running manual cleanup instead: {e}")
|
||||
|
||||
# Manual cleanup
|
||||
print("🧹 Running manual cleanup...")
|
||||
cleaned_items = []
|
||||
|
||||
# Remove Python cache
|
||||
for cache_dir in root_path.rglob('__pycache__'):
|
||||
if cache_dir.is_dir():
|
||||
try:
|
||||
shutil.rmtree(cache_dir)
|
||||
cleaned_items.append(f"Removed: {cache_dir.relative_to(root_path)}")
|
||||
except:
|
||||
pass
|
||||
|
||||
# Remove temp files
|
||||
patterns = ['*.pyc', '*.pyo', '*_temp*', '*.tmp', '*~']
|
||||
for pattern in patterns:
|
||||
for file_path in root_path.rglob(pattern):
|
||||
if file_path.is_file() and '.git' not in str(file_path):
|
||||
try:
|
||||
file_path.unlink()
|
||||
cleaned_items.append(f"Removed: {file_path.relative_to(root_path)}")
|
||||
except:
|
||||
pass
|
||||
|
||||
# Organize logs
|
||||
logs_dir = root_path / 'reports' / 'logs'
|
||||
logs_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for log_file in root_path.glob('*.log'):
|
||||
try:
|
||||
shutil.move(str(log_file), str(logs_dir / log_file.name))
|
||||
cleaned_items.append(f"Moved: {log_file.name} → reports/logs/")
|
||||
except:
|
||||
pass
|
||||
|
||||
# Step 5: Setup git hooks (optional)
|
||||
print("\n" + "=" * 50)
|
||||
print("🔧 OPTIONAL GIT SETUP")
|
||||
print("=" * 50)
|
||||
|
||||
# Check if this is a git repository
|
||||
if (root_path / '.git').exists():
|
||||
print("📝 Setting up git configuration...")
|
||||
|
||||
# Add files to git
|
||||
run_command("git add .gitignore", "Adding .gitignore")
|
||||
run_command("git add scripts/quick_cleanup.py", "Adding cleanup script")
|
||||
run_command("git add .forgejo/workflows/generate-m3u.yml", "Adding enhanced workflow")
|
||||
|
||||
print(" ✅ Files staged for commit")
|
||||
else:
|
||||
print(" ℹ️ Not a git repository, skipping git setup")
|
||||
|
||||
# Step 6: Final summary
|
||||
print("\n" + "=" * 50)
|
||||
print("✅ SETUP COMPLETE!")
|
||||
print("=" * 50)
|
||||
|
||||
print(f"📊 Summary:")
|
||||
print(f" - Cleaned {len(cleaned_items)} items")
|
||||
print(f" - Created enhanced .gitignore")
|
||||
print(f" - Added quick cleanup script")
|
||||
print(f" - Updated Forgejo workflow")
|
||||
print(f" - Organized directory structure")
|
||||
|
||||
print(f"\n🎯 What's New:")
|
||||
print(f" 📁 reports/logs/ - All logs organized here")
|
||||
print(f" 🧹 scripts/quick_cleanup.py - Manual cleanup tool")
|
||||
print(f" 🔄 Enhanced workflow - Automatic cleanup on push")
|
||||
print(f" 🚫 .gitignore - Prevents future clutter")
|
||||
|
||||
print(f"\n🚀 Next Steps:")
|
||||
print(f" 1. Commit these changes: git commit -m 'Setup automated cleanup'")
|
||||
print(f" 2. Push to trigger workflow: git push")
|
||||
print(f" 3. For manual cleanup: python scripts/quick_cleanup.py")
|
||||
print(f" 4. Your repository will now stay clean automatically!")
|
||||
|
||||
print(f"\n🎉 Your IPTV repository is now clean and organized!")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue