Add status webapp
This commit is contained in:
parent
c25f1daf76
commit
266558bb15
462
aocsync.py
462
aocsync.py
@ -14,12 +14,19 @@ import shutil
|
||||
import re
|
||||
import time
|
||||
import logging
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from dataclasses import dataclass, asdict
|
||||
from collections import defaultdict
|
||||
|
||||
try:
|
||||
from flask import Flask, Response, jsonify
|
||||
FLASK_AVAILABLE = True
|
||||
except ImportError:
|
||||
FLASK_AVAILABLE = False
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
@ -2338,8 +2345,12 @@ class AOCSync:
|
||||
except Exception as e:
|
||||
logger.error(f"Error building Podman image: {e}")
|
||||
|
||||
def sync_all(self):
|
||||
def sync_all(self, force: bool = None):
|
||||
"""Sync all repositories"""
|
||||
if force is not None:
|
||||
original_force = self.force_rerun
|
||||
self.force_rerun = force
|
||||
|
||||
logger.info("Starting sync of all repositories...")
|
||||
|
||||
# Clear log file at start of sync
|
||||
@ -2365,6 +2376,102 @@ class AOCSync:
|
||||
# Rsync output if configured
|
||||
self._rsync_output()
|
||||
|
||||
if force is not None:
|
||||
self.force_rerun = original_force
|
||||
|
||||
def sync_repo(self, repo_name: str, force: bool = True):
|
||||
"""Sync a specific repository by name"""
|
||||
logger.info(f"Starting sync for repository: {repo_name} (force={force})...")
|
||||
|
||||
original_force = self.force_rerun
|
||||
self.force_rerun = force
|
||||
|
||||
# Append to log file instead of clearing
|
||||
log_file = Path(self.config.output_dir) / 'cargo-aoc.log'
|
||||
log_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(log_file, 'a', encoding='utf-8') as f:
|
||||
f.write(f"\n{'#'*80}\n")
|
||||
f.write(f"# Sync started for {repo_name} at {datetime.now().isoformat()}\n")
|
||||
f.write(f"{'#'*80}\n\n")
|
||||
|
||||
found = False
|
||||
for repo_config in self.config.repositories:
|
||||
if repo_config['name'] == repo_name:
|
||||
found = True
|
||||
try:
|
||||
self.process_repository(repo_config, repo_name)
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing repository {repo_name}: {e}")
|
||||
break
|
||||
|
||||
if not found:
|
||||
logger.error(f"Repository {repo_name} not found")
|
||||
|
||||
# Generate HTML report
|
||||
logger.info("Generating HTML report...")
|
||||
self.html_gen.generate(self.db, self.config)
|
||||
|
||||
# Rsync output if configured
|
||||
self._rsync_output()
|
||||
|
||||
self.force_rerun = original_force
|
||||
|
||||
def sync_year(self, year: int, force: bool = True):
|
||||
"""Sync all repositories for a specific year"""
|
||||
logger.info(f"Starting sync for year: {year} (force={force})...")
|
||||
|
||||
original_force = self.force_rerun
|
||||
self.force_rerun = force
|
||||
|
||||
# Append to log file instead of clearing
|
||||
log_file = Path(self.config.output_dir) / 'cargo-aoc.log'
|
||||
log_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(log_file, 'a', encoding='utf-8') as f:
|
||||
f.write(f"\n{'#'*80}\n")
|
||||
f.write(f"# Sync started for year {year} at {datetime.now().isoformat()}\n")
|
||||
f.write(f"{'#'*80}\n\n")
|
||||
|
||||
for repo_config in self.config.repositories:
|
||||
user_name = repo_config['name']
|
||||
repo_type = repo_config.get('type', 'single')
|
||||
|
||||
try:
|
||||
if repo_type == 'single':
|
||||
# Check if this repo has the year
|
||||
url = repo_config['url']
|
||||
local_path = repo_config['local_path']
|
||||
if self.git_manager.clone_or_update_repo(url, local_path):
|
||||
repo_path = Path(local_path)
|
||||
config_years = repo_config.get('years', [])
|
||||
years_to_process = config_years if config_years else CargoAOCRunner.extract_years_from_repo(repo_path)
|
||||
|
||||
if year in years_to_process:
|
||||
logger.info(f"Processing {user_name} year {year}...")
|
||||
self._run_and_store_benchmarks(repo_path, year, user_name,
|
||||
repo_url=url, is_multi_year=False)
|
||||
|
||||
elif repo_type == 'multi-year':
|
||||
years_config = repo_config.get('years', [])
|
||||
for year_config in years_config:
|
||||
if year_config['year'] == year:
|
||||
url = year_config['url']
|
||||
local_path = year_config['local_path']
|
||||
if self.git_manager.clone_or_update_repo(url, local_path):
|
||||
repo_path = Path(local_path)
|
||||
self._run_and_store_benchmarks(repo_path, year, user_name,
|
||||
repo_url=url, is_multi_year=True)
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing repository {user_name} for year {year}: {e}")
|
||||
|
||||
# Generate HTML report
|
||||
logger.info("Generating HTML report...")
|
||||
self.html_gen.generate(self.db, self.config)
|
||||
|
||||
# Rsync output if configured
|
||||
self._rsync_output()
|
||||
|
||||
self.force_rerun = original_force
|
||||
|
||||
def _rsync_output(self):
|
||||
"""Rsync output directory to remote server if configured"""
|
||||
rsync_config = self.config.rsync_config
|
||||
@ -2419,6 +2526,333 @@ class AOCSync:
|
||||
logger.info("Stopped by user")
|
||||
|
||||
|
||||
class WebServer:
|
||||
"""Simple web server for viewing logs and triggering refreshes"""
|
||||
|
||||
def __init__(self, sync: AOCSync, host: str = '0.0.0.0', port: int = 8080):
|
||||
self.sync = sync
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.app = None
|
||||
self._setup_app()
|
||||
|
||||
def _setup_app(self):
|
||||
"""Setup Flask application"""
|
||||
if not FLASK_AVAILABLE:
|
||||
raise ImportError("Flask is required for web server. Install with: pip install Flask")
|
||||
self.app = Flask(__name__)
|
||||
|
||||
@self.app.route('/')
|
||||
def index():
|
||||
return self._get_index_page()
|
||||
|
||||
@self.app.route('/logs')
|
||||
def logs():
|
||||
"""View logs"""
|
||||
log_file = Path(self.sync.config.output_dir) / 'cargo-aoc.log'
|
||||
if log_file.exists():
|
||||
with open(log_file, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
return Response(content, mimetype='text/plain')
|
||||
return "No log file found", 404
|
||||
|
||||
@self.app.route('/api/refresh/all', methods=['POST'])
|
||||
def refresh_all():
|
||||
"""Trigger refresh for all repositories"""
|
||||
thread = threading.Thread(target=self.sync.sync_all, kwargs={'force': True})
|
||||
thread.daemon = True
|
||||
thread.start()
|
||||
return jsonify({'status': 'started', 'message': 'Refresh started for all repositories'})
|
||||
|
||||
@self.app.route('/api/refresh/repo/<repo_name>', methods=['POST'])
|
||||
def refresh_repo(repo_name):
|
||||
"""Trigger refresh for a specific repository"""
|
||||
thread = threading.Thread(target=self.sync.sync_repo, args=(repo_name,), kwargs={'force': True})
|
||||
thread.daemon = True
|
||||
thread.start()
|
||||
return jsonify({'status': 'started', 'message': f'Refresh started for repository: {repo_name}'})
|
||||
|
||||
@self.app.route('/api/refresh/year/<int:year>', methods=['POST'])
|
||||
def refresh_year(year):
|
||||
"""Trigger refresh for a specific year"""
|
||||
thread = threading.Thread(target=self.sync.sync_year, args=(year,), kwargs={'force': True})
|
||||
thread.daemon = True
|
||||
thread.start()
|
||||
return jsonify({'status': 'started', 'message': f'Refresh started for year: {year}'})
|
||||
|
||||
@self.app.route('/api/repos', methods=['GET'])
|
||||
def get_repos():
|
||||
"""Get list of repositories"""
|
||||
repos = [{'name': r['name'], 'type': r.get('type', 'single')} for r in self.sync.config.repositories]
|
||||
return jsonify({'repos': repos})
|
||||
|
||||
@self.app.route('/api/years', methods=['GET'])
|
||||
def get_years():
|
||||
"""Get list of years"""
|
||||
years = self.sync.db.get_all_years()
|
||||
return jsonify({'years': sorted(years, reverse=True)})
|
||||
|
||||
def _get_index_page(self):
|
||||
"""Generate the main web interface page"""
|
||||
repos = [r['name'] for r in self.sync.config.repositories]
|
||||
years = sorted(self.sync.db.get_all_years(), reverse=True)
|
||||
|
||||
# Build HTML - use regular string and format variables manually to avoid brace escaping issues
|
||||
html = """<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>AOC Sync Control Panel</title>
|
||||
<style>
|
||||
* {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
body {
|
||||
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif;
|
||||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||||
min-height: 100vh;
|
||||
padding: 20px;
|
||||
}
|
||||
|
||||
.container {
|
||||
max-width: 1200px;
|
||||
margin: 0 auto;
|
||||
background: white;
|
||||
border-radius: 8px;
|
||||
box-shadow: 0 10px 30px rgba(0,0,0,0.2);
|
||||
padding: 30px;
|
||||
}
|
||||
|
||||
h1 {
|
||||
color: #333;
|
||||
margin-bottom: 10px;
|
||||
font-size: 2em;
|
||||
}
|
||||
|
||||
.section {
|
||||
margin-bottom: 30px;
|
||||
padding: 20px;
|
||||
background: #f8f9fa;
|
||||
border-radius: 6px;
|
||||
}
|
||||
|
||||
.section h2 {
|
||||
color: #667eea;
|
||||
margin-bottom: 15px;
|
||||
font-size: 1.3em;
|
||||
}
|
||||
|
||||
.button-group {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 10px;
|
||||
margin-bottom: 15px;
|
||||
}
|
||||
|
||||
button {
|
||||
padding: 10px 20px;
|
||||
background: #667eea;
|
||||
color: white;
|
||||
border: none;
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
font-size: 0.9em;
|
||||
transition: background 0.2s;
|
||||
}
|
||||
|
||||
button:hover {
|
||||
background: #5568d3;
|
||||
}
|
||||
|
||||
button:disabled {
|
||||
background: #ccc;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
.button-danger {
|
||||
background: #dc3545;
|
||||
}
|
||||
|
||||
.button-danger:hover {
|
||||
background: #c82333;
|
||||
}
|
||||
|
||||
.status {
|
||||
margin-top: 10px;
|
||||
padding: 10px;
|
||||
border-radius: 4px;
|
||||
display: none;
|
||||
}
|
||||
|
||||
.status.success {
|
||||
background: #d4edda;
|
||||
color: #155724;
|
||||
border: 1px solid #c3e6cb;
|
||||
}
|
||||
|
||||
.status.error {
|
||||
background: #f8d7da;
|
||||
color: #721c24;
|
||||
border: 1px solid #f5c6cb;
|
||||
}
|
||||
|
||||
.log-viewer {
|
||||
background: #1e1e1e;
|
||||
color: #d4d4d4;
|
||||
padding: 15px;
|
||||
border-radius: 4px;
|
||||
font-family: 'Courier New', monospace;
|
||||
font-size: 0.85em;
|
||||
max-height: 500px;
|
||||
overflow-y: auto;
|
||||
white-space: pre-wrap;
|
||||
word-wrap: break-word;
|
||||
}
|
||||
|
||||
.log-viewer a {
|
||||
color: #4ec9b0;
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
.log-viewer a:hover {
|
||||
text-decoration: underline;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<h1>🎄 AOC Sync Control Panel</h1>
|
||||
|
||||
<div class="section">
|
||||
<h2>Refresh Controls</h2>
|
||||
|
||||
<div style="margin-bottom: 20px;">
|
||||
<h3 style="margin-bottom: 10px; color: #555;">Refresh All</h3>
|
||||
<button onclick="refreshAll()" id="btn-refresh-all">🔄 Refresh All Repositories</button>
|
||||
<div class="status" id="status-all"></div>
|
||||
</div>
|
||||
|
||||
<div style="margin-bottom: 20px;">
|
||||
<h3 style="margin-bottom: 10px; color: #555;">Refresh by Repository</h3>
|
||||
<div class="button-group">
|
||||
"""
|
||||
for repo in repos:
|
||||
# Use double quotes for outer string, single quotes for JavaScript
|
||||
html += f" <button onclick=\"refreshRepo('{repo}')\" id=\"btn-repo-{repo}\">🔄 {repo}</button>\n"
|
||||
|
||||
html += """ </div>
|
||||
<div class="status" id="status-repo"></div>
|
||||
</div>
|
||||
|
||||
<div style="margin-bottom: 20px;">
|
||||
<h3 style="margin-bottom: 10px; color: #555;">Refresh by Year</h3>
|
||||
<div class="button-group">
|
||||
"""
|
||||
for year in years:
|
||||
html += f' <button onclick="refreshYear({year})" id="btn-year-{year}">🔄 {year}</button>\n'
|
||||
|
||||
html += """ </div>
|
||||
<div class="status" id="status-year"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="section">
|
||||
<h2>Logs</h2>
|
||||
<p style="margin-bottom: 10px; color: #666;">
|
||||
<a href="/logs" target="_blank" style="color: #667eea;">📋 View Full Logs</a>
|
||||
</p>
|
||||
<div class="log-viewer" id="log-viewer">Loading logs...</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
function showStatus(elementId, message, isError = false) {
|
||||
const status = document.getElementById(elementId);
|
||||
status.textContent = message;
|
||||
status.className = 'status ' + (isError ? 'error' : 'success');
|
||||
status.style.display = 'block';
|
||||
setTimeout(() => {
|
||||
status.style.display = 'none';
|
||||
}, 5000);
|
||||
}
|
||||
|
||||
async function refreshAll() {
|
||||
const btn = document.getElementById('btn-refresh-all');
|
||||
btn.disabled = true;
|
||||
try {
|
||||
const response = await fetch('/api/refresh/all', { method: 'POST' });
|
||||
const data = await response.json();
|
||||
showStatus('status-all', data.message || 'Refresh started', false);
|
||||
} catch (error) {
|
||||
showStatus('status-all', 'Error: ' + error.message, true);
|
||||
} finally {
|
||||
btn.disabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshRepo(repoName) {
|
||||
const btn = document.getElementById('btn-repo-' + repoName);
|
||||
btn.disabled = true;
|
||||
try {
|
||||
const response = await fetch('/api/refresh/repo/' + encodeURIComponent(repoName), { method: 'POST' });
|
||||
const data = await response.json();
|
||||
showStatus('status-repo', data.message || 'Refresh started', false);
|
||||
} catch (error) {
|
||||
showStatus('status-repo', 'Error: ' + error.message, true);
|
||||
} finally {
|
||||
btn.disabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshYear(year) {
|
||||
const btn = document.getElementById('btn-year-' + year);
|
||||
btn.disabled = true;
|
||||
try {
|
||||
const response = await fetch('/api/refresh/year/' + year, { method: 'POST' });
|
||||
const data = await response.json();
|
||||
showStatus('status-year', data.message || 'Refresh started', false);
|
||||
} catch (error) {
|
||||
showStatus('status-year', 'Error: ' + error.message, true);
|
||||
} finally {
|
||||
btn.disabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function loadLogs() {
|
||||
try {
|
||||
const response = await fetch('/logs');
|
||||
if (response.ok) {
|
||||
const text = await response.text();
|
||||
const logViewer = document.getElementById('log-viewer');
|
||||
// Show last 5000 characters
|
||||
logViewer.textContent = text.slice(-5000);
|
||||
logViewer.scrollTop = logViewer.scrollHeight;
|
||||
} else {
|
||||
document.getElementById('log-viewer').textContent = 'No logs available';
|
||||
}
|
||||
} catch (error) {
|
||||
document.getElementById('log-viewer').textContent = 'Error loading logs: ' + error.message;
|
||||
}
|
||||
}
|
||||
|
||||
// Load logs on page load and refresh every 5 seconds
|
||||
loadLogs();
|
||||
setInterval(loadLogs, 5000);
|
||||
</script>
|
||||
</body>
|
||||
</html>"""
|
||||
return html
|
||||
|
||||
def run(self):
|
||||
"""Run the web server"""
|
||||
logger.info(f"Starting web server on http://{self.host}:{self.port}")
|
||||
self.app.run(host=self.host, port=self.port, debug=False, use_reloader=False)
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point"""
|
||||
import argparse
|
||||
@ -2428,15 +2862,35 @@ def main():
|
||||
parser.add_argument('--once', action='store_true', help='Run once instead of continuously')
|
||||
parser.add_argument('--force', '--rerun-all', action='store_true', dest='force_rerun',
|
||||
help='Force rerun all days even if repository has not changed')
|
||||
parser.add_argument('--web', action='store_true', help='Start web server for logs and refresh controls')
|
||||
parser.add_argument('--web-host', default='0.0.0.0', help='Web server host (default: 0.0.0.0)')
|
||||
parser.add_argument('--web-port', type=int, default=8080, help='Web server port (default: 8080)')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
sync = AOCSync(args.config, force_rerun=args.force_rerun)
|
||||
|
||||
if args.once:
|
||||
sync.sync_all()
|
||||
if args.web:
|
||||
if not FLASK_AVAILABLE:
|
||||
logger.error("Flask is required for web server. Install with: pip install Flask")
|
||||
sys.exit(1)
|
||||
# Start web server
|
||||
web_server = WebServer(sync, host=args.web_host, port=args.web_port)
|
||||
if args.once:
|
||||
# Run once then start web server
|
||||
sync.sync_all()
|
||||
web_server.run()
|
||||
else:
|
||||
# Start web server in background thread, then run continuous sync
|
||||
web_thread = threading.Thread(target=web_server.run)
|
||||
web_thread.daemon = True
|
||||
web_thread.start()
|
||||
sync.run_continuous()
|
||||
else:
|
||||
sync.run_continuous()
|
||||
if args.once:
|
||||
sync.sync_all()
|
||||
else:
|
||||
sync.run_continuous()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
@ -1 +1,2 @@
|
||||
PyYAML>=6.0
|
||||
Flask>=2.0.0
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user