|
|
|
|
@@ -14,12 +14,19 @@ import shutil
|
|
|
|
|
import re
|
|
|
|
|
import time
|
|
|
|
|
import logging
|
|
|
|
|
import threading
|
|
|
|
|
from pathlib import Path
|
|
|
|
|
from datetime import datetime
|
|
|
|
|
from typing import Dict, List, Optional, Tuple
|
|
|
|
|
from dataclasses import dataclass, asdict
|
|
|
|
|
from collections import defaultdict
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
from flask import Flask, Response, jsonify
|
|
|
|
|
FLASK_AVAILABLE = True
|
|
|
|
|
except ImportError:
|
|
|
|
|
FLASK_AVAILABLE = False
|
|
|
|
|
|
|
|
|
|
# Configure logging
|
|
|
|
|
logging.basicConfig(
|
|
|
|
|
level=logging.INFO,
|
|
|
|
|
@@ -577,7 +584,7 @@ class CargoAOCRunner:
|
|
|
|
|
return commits
|
|
|
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
|
def _run_cargo_aoc_in_container(work_dir: Path, day: int, repo_root: Path, docker_config: dict) -> subprocess.CompletedProcess:
|
|
|
|
|
def _run_cargo_aoc_in_container(work_dir: Path, day: int, repo_root: Path, docker_config: dict, user: str = "unknown", year: int = 0) -> subprocess.CompletedProcess:
|
|
|
|
|
"""Run cargo aoc in a Podman container for security
|
|
|
|
|
|
|
|
|
|
Args:
|
|
|
|
|
@@ -585,6 +592,8 @@ class CargoAOCRunner:
|
|
|
|
|
day: Day number to run
|
|
|
|
|
repo_root: Absolute path to repository root
|
|
|
|
|
docker_config: Podman configuration dictionary
|
|
|
|
|
user: User name for build cache directory organization
|
|
|
|
|
year: Year for build cache directory organization
|
|
|
|
|
|
|
|
|
|
Returns:
|
|
|
|
|
CompletedProcess with stdout, stderr, returncode
|
|
|
|
|
@@ -605,14 +614,16 @@ class CargoAOCRunner:
|
|
|
|
|
temp_build_dir = None
|
|
|
|
|
|
|
|
|
|
if build_cache_dir:
|
|
|
|
|
# Use persistent build cache directory
|
|
|
|
|
build_cache_path = Path(build_cache_dir).resolve()
|
|
|
|
|
# Use persistent build cache directory with user/year subdirectories
|
|
|
|
|
base_cache_path = Path(build_cache_dir).resolve()
|
|
|
|
|
# Create user/year specific directory: build_cache_dir/user/year/
|
|
|
|
|
build_cache_path = base_cache_path / user / str(year)
|
|
|
|
|
build_cache_path.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
logger.info(f"Using persistent build cache: {build_cache_path}")
|
|
|
|
|
logger.info(f"Using persistent build cache: {build_cache_path} (user: {user}, year: {year})")
|
|
|
|
|
else:
|
|
|
|
|
# Create a temporary directory for cargo build artifacts (outside repo)
|
|
|
|
|
import tempfile
|
|
|
|
|
temp_build_dir = tempfile.mkdtemp(prefix='cargo-aoc-build-')
|
|
|
|
|
temp_build_dir = tempfile.mkdtemp(prefix=f'cargo-aoc-build-{user}-{year}-')
|
|
|
|
|
build_cache_path = Path(temp_build_dir)
|
|
|
|
|
use_temp_build = True
|
|
|
|
|
logger.info(f"Using temporary build cache: {build_cache_path}")
|
|
|
|
|
@@ -670,13 +681,13 @@ class CargoAOCRunner:
|
|
|
|
|
# Check if already installed to avoid reinstalling every time
|
|
|
|
|
podman_cmd.extend([
|
|
|
|
|
podman_image,
|
|
|
|
|
'sh', '-c', 'if ! command -v cargo-aoc >/dev/null 2>&1; then cargo install --quiet cargo-aoc 2>/dev/null || true; fi; cargo aoc --day ' + str(day)
|
|
|
|
|
'sh', '-c', f'if ! command -v cargo-aoc >/dev/null 2>&1; then cargo install --quiet --git https://github.com/ggriffiniii/cargo-aoc cargo-aoc 2>/dev/null || true; fi; cargo aoc bench -d {day} -- --quick'
|
|
|
|
|
])
|
|
|
|
|
else:
|
|
|
|
|
# Use pre-installed cargo-aoc (faster, requires aocsync:latest image)
|
|
|
|
|
podman_cmd.extend([
|
|
|
|
|
podman_image,
|
|
|
|
|
'cargo', 'aoc', '--day', str(day)
|
|
|
|
|
'cargo', 'aoc', 'bench', '-d', str(day), '--', '--quick'
|
|
|
|
|
])
|
|
|
|
|
|
|
|
|
|
result = subprocess.run(
|
|
|
|
|
@@ -740,7 +751,7 @@ class CargoAOCRunner:
|
|
|
|
|
|
|
|
|
|
for day in days:
|
|
|
|
|
try:
|
|
|
|
|
logger.info(f"Running cargo aoc for {user} year {year} day {day} in {work_dir} (in Podman container)")
|
|
|
|
|
logger.info(f"Running cargo aoc bench for {user} year {year} day {day} in {work_dir} (in Podman container)")
|
|
|
|
|
# Run cargo aoc in a Podman container for security
|
|
|
|
|
# Use default docker_config if not provided
|
|
|
|
|
if docker_config is None:
|
|
|
|
|
@@ -751,7 +762,7 @@ class CargoAOCRunner:
|
|
|
|
|
'cpus': '2',
|
|
|
|
|
'image': 'aocsync:latest'
|
|
|
|
|
}
|
|
|
|
|
result = CargoAOCRunner._run_cargo_aoc_in_container(work_dir, day, repo_root, docker_config)
|
|
|
|
|
result = CargoAOCRunner._run_cargo_aoc_in_container(work_dir, day, repo_root, docker_config, user, year)
|
|
|
|
|
|
|
|
|
|
# Write to log file if provided
|
|
|
|
|
if log_file:
|
|
|
|
|
@@ -763,7 +774,7 @@ class CargoAOCRunner:
|
|
|
|
|
with open(log_file, 'a', encoding='utf-8') as f:
|
|
|
|
|
f.write(f"\n{'='*80}\n")
|
|
|
|
|
f.write(f"[{timestamp}] {user} - Year {year} - Day {day}\n")
|
|
|
|
|
f.write(f"Command: cargo aoc --day {day} (in Podman container)\n")
|
|
|
|
|
f.write(f"Command: cargo aoc bench -d {day} -- --quick (in Podman container)\n")
|
|
|
|
|
f.write(f"Working Directory: {work_dir}\n")
|
|
|
|
|
f.write(f"Return Code: {result.returncode}\n")
|
|
|
|
|
f.write(f"{'='*80}\n")
|
|
|
|
|
@@ -778,12 +789,14 @@ class CargoAOCRunner:
|
|
|
|
|
f.write(f"{'='*80}\n\n")
|
|
|
|
|
|
|
|
|
|
if result.returncode != 0:
|
|
|
|
|
logger.warning(f"cargo aoc failed for day {day} in {work_dir}: {result.stderr}")
|
|
|
|
|
continue
|
|
|
|
|
logger.warning(f"cargo aoc bench failed for day {day} in {work_dir} (return code: {result.returncode}). Will still attempt to parse any available timing data.")
|
|
|
|
|
|
|
|
|
|
# Log output for debugging if no results found
|
|
|
|
|
if not result.stdout.strip() and not result.stderr.strip():
|
|
|
|
|
logger.warning(f"No output from cargo aoc for {user} year {year} day {day}")
|
|
|
|
|
logger.warning(f"No output from cargo aoc bench for {user} year {year} day {day}")
|
|
|
|
|
# Skip parsing if there's no output at all
|
|
|
|
|
if result.returncode != 0:
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
# Strip ANSI codes before parsing (for cleaner parsing)
|
|
|
|
|
stdout_clean = CargoAOCRunner._strip_ansi_codes(result.stdout or "")
|
|
|
|
|
@@ -793,26 +806,34 @@ class CargoAOCRunner:
|
|
|
|
|
output_bytes = len(result.stdout.encode('utf-8')) if result.stdout else 0
|
|
|
|
|
|
|
|
|
|
# Parse output for runtime information
|
|
|
|
|
# Even if return code is non-zero (e.g., Part 2 panics), Part 1 timing might still be in output
|
|
|
|
|
day_results = CargoAOCRunner._parse_runtime_output(
|
|
|
|
|
stdout_clean, stderr_clean, day, year, user, git_rev, repo_url, output_bytes
|
|
|
|
|
)
|
|
|
|
|
if day_results:
|
|
|
|
|
logger.info(f"Parsed {len(day_results)} runtime result(s) for {user} year {year} day {day}")
|
|
|
|
|
if result.returncode != 0:
|
|
|
|
|
# Log which parts were successfully parsed despite the error
|
|
|
|
|
parts_parsed = [f"Part {r.part}" for r in day_results]
|
|
|
|
|
logger.info(f"Successfully parsed timing for {', '.join(parts_parsed)} despite non-zero return code")
|
|
|
|
|
else:
|
|
|
|
|
# Log a sample of the output to help debug parsing issues
|
|
|
|
|
output_sample = (result.stdout + "\n" + result.stderr).strip()[:500]
|
|
|
|
|
logger.warning(f"No runtime data parsed for {user} year {year} day {day}. Output sample: {output_sample}")
|
|
|
|
|
# Only skip if we got no results AND there was an error
|
|
|
|
|
if result.returncode != 0:
|
|
|
|
|
continue
|
|
|
|
|
results.extend(day_results)
|
|
|
|
|
|
|
|
|
|
except subprocess.TimeoutExpired:
|
|
|
|
|
error_msg = f"Timeout running cargo aoc for day {day}"
|
|
|
|
|
error_msg = f"Timeout running cargo aoc bench for day {day}"
|
|
|
|
|
logger.error(error_msg)
|
|
|
|
|
if log_file:
|
|
|
|
|
timestamp = datetime.now().isoformat()
|
|
|
|
|
with open(log_file, 'a', encoding='utf-8') as f:
|
|
|
|
|
f.write(f"\n[{timestamp}] ERROR: {error_msg}\n")
|
|
|
|
|
except Exception as e:
|
|
|
|
|
error_msg = f"Error running cargo aoc for day {day}: {e}"
|
|
|
|
|
error_msg = f"Error running cargo aoc bench for day {day}: {e}"
|
|
|
|
|
logger.error(error_msg)
|
|
|
|
|
if log_file:
|
|
|
|
|
timestamp = datetime.now().isoformat()
|
|
|
|
|
@@ -824,9 +845,14 @@ class CargoAOCRunner:
|
|
|
|
|
@staticmethod
|
|
|
|
|
def _parse_runtime_output(stdout: str, stderr: str, day: int, year: int,
|
|
|
|
|
user: str, git_rev: str = "", repo_url: str = "", output_bytes: int = 0) -> List[PerformanceResult]:
|
|
|
|
|
"""Parse cargo-aoc runtime output
|
|
|
|
|
"""Parse cargo aoc bench runtime output
|
|
|
|
|
|
|
|
|
|
cargo aoc typically outputs timing information like:
|
|
|
|
|
cargo aoc bench -- --quick outputs timing information in format like:
|
|
|
|
|
- "Day8 - Part1/(default) time: [15.127 ms 15.168 ms 15.331 ms]"
|
|
|
|
|
- "Day8 - Part2/(default) time: [15.141 ms 15.160 ms 15.164 ms]"
|
|
|
|
|
Extracts the middle measurement (second value) from the three measurements.
|
|
|
|
|
|
|
|
|
|
Also supports legacy cargo-aoc custom format for backward compatibility:
|
|
|
|
|
- "Day X - Part Y: XXX.XXX ms"
|
|
|
|
|
- "Day X - Part Y: XXX.XXX μs"
|
|
|
|
|
- "Day X - Part Y: XXX.XXX ns"
|
|
|
|
|
@@ -839,6 +865,43 @@ class CargoAOCRunner:
|
|
|
|
|
# Combine stdout and stderr (timing info might be in either)
|
|
|
|
|
output = stdout + "\n" + stderr
|
|
|
|
|
|
|
|
|
|
# First, try to parse cargo aoc bench output format
|
|
|
|
|
# Pattern: Day<X> - Part<Y>/(default) time: [<val1> <unit> <val2> <unit> <val3> <unit>]
|
|
|
|
|
# Example: Day8 - Part1/(default) time: [15.127 ms 15.168 ms 15.331 ms]
|
|
|
|
|
# Extract the middle measurement (val2)
|
|
|
|
|
cargo_bench_pattern = r'Day\s*(\d+)\s*-\s*Part\s*(\d+).*?time:\s*\[([\d.]+)\s+(\w+)\s+([\d.]+)\s+(\w+)\s+([\d.]+)\s+(\w+)\]'
|
|
|
|
|
for match in re.finditer(cargo_bench_pattern, output, re.IGNORECASE | re.MULTILINE):
|
|
|
|
|
bench_day = int(match.group(1))
|
|
|
|
|
bench_part = int(match.group(2))
|
|
|
|
|
# Extract middle value (group 5) and its unit (group 6)
|
|
|
|
|
time_str = match.group(5)
|
|
|
|
|
unit = match.group(6).lower()
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
time_val = float(time_str)
|
|
|
|
|
time_ns = CargoAOCRunner._convert_to_nanoseconds(time_val, unit)
|
|
|
|
|
|
|
|
|
|
# Avoid duplicates
|
|
|
|
|
if not any(r.day == bench_day and r.part == bench_part
|
|
|
|
|
for r in results):
|
|
|
|
|
results.append(PerformanceResult(
|
|
|
|
|
user=user,
|
|
|
|
|
year=year,
|
|
|
|
|
day=bench_day,
|
|
|
|
|
part=bench_part,
|
|
|
|
|
time_ns=time_ns,
|
|
|
|
|
output_bytes=output_bytes,
|
|
|
|
|
git_rev=git_rev,
|
|
|
|
|
repo_url=repo_url,
|
|
|
|
|
timestamp=timestamp
|
|
|
|
|
))
|
|
|
|
|
except ValueError:
|
|
|
|
|
logger.warning(f"Could not parse cargo bench time: {time_str} {unit}")
|
|
|
|
|
|
|
|
|
|
# If we found results with cargo bench format, return them
|
|
|
|
|
if results:
|
|
|
|
|
return results
|
|
|
|
|
|
|
|
|
|
# Patterns to match various cargo-aoc output formats
|
|
|
|
|
# Common formats:
|
|
|
|
|
# "Day 1 - Part 1: 123.456 ms"
|
|
|
|
|
@@ -1929,9 +1992,10 @@ class HTMLGenerator:
|
|
|
|
|
speed_multiple = speed_multiples.get((day, part), 0)
|
|
|
|
|
row_history_modals = []
|
|
|
|
|
|
|
|
|
|
aoc_url = f"https://adventofcode.com/{year}/day/{day}"
|
|
|
|
|
html += f"""
|
|
|
|
|
<tr>
|
|
|
|
|
<td><strong>Day {day} Part {part}</strong></td>
|
|
|
|
|
<td><strong><a href="{aoc_url}" target="_blank">Day {day} Part {part}</a></strong></td>
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
# Add timing data for each user
|
|
|
|
|
@@ -2091,9 +2155,10 @@ class HTMLGenerator:
|
|
|
|
|
continue
|
|
|
|
|
for day in sorted(data[year].keys()):
|
|
|
|
|
for part in sorted(data[year][day].keys()):
|
|
|
|
|
aoc_url = f"https://adventofcode.com/{year}/day/{day}"
|
|
|
|
|
html += f""" <tr>
|
|
|
|
|
<td>{year}</td>
|
|
|
|
|
<td>{day}</td>
|
|
|
|
|
<td><a href="{aoc_url}" target="_blank">{day}</a></td>
|
|
|
|
|
<td>{part}</td>
|
|
|
|
|
"""
|
|
|
|
|
for user in sorted(users):
|
|
|
|
|
@@ -2220,20 +2285,32 @@ class AOCSync:
|
|
|
|
|
elif repo_type == 'multi-year':
|
|
|
|
|
# Multiple repositories, one per year
|
|
|
|
|
years_config = repo_config.get('years', [])
|
|
|
|
|
if not years_config:
|
|
|
|
|
logger.warning(f"No years configured for multi-year repository {user_name}")
|
|
|
|
|
return
|
|
|
|
|
|
|
|
|
|
logger.info(f"Processing multi-year repository {user_name} with {len(years_config)} year(s)")
|
|
|
|
|
for year_config in years_config:
|
|
|
|
|
year = year_config['year']
|
|
|
|
|
url = year_config['url']
|
|
|
|
|
local_path = year_config['local_path']
|
|
|
|
|
|
|
|
|
|
if self.force_rerun or self.git_manager.has_changes(url, local_path):
|
|
|
|
|
logger.debug(f"Checking {user_name} year {year} at {local_path}")
|
|
|
|
|
has_changes_result = GitManager.has_changes(url, local_path)
|
|
|
|
|
|
|
|
|
|
if self.force_rerun or has_changes_result:
|
|
|
|
|
if self.force_rerun:
|
|
|
|
|
logger.info(f"Force rerun enabled, processing repository {user_name} year {year}...")
|
|
|
|
|
else:
|
|
|
|
|
logger.info(f"Repository {user_name} year {year} has changes, updating...")
|
|
|
|
|
if self.git_manager.clone_or_update_repo(url, local_path):
|
|
|
|
|
if GitManager.clone_or_update_repo(url, local_path):
|
|
|
|
|
repo_path = Path(local_path)
|
|
|
|
|
self._run_and_store_benchmarks(repo_path, year, user_name,
|
|
|
|
|
repo_url=url, is_multi_year=True)
|
|
|
|
|
else:
|
|
|
|
|
logger.error(f"Failed to clone/update repository {user_name} year {year} at {local_path}")
|
|
|
|
|
else:
|
|
|
|
|
logger.info(f"Repository {user_name} year {year} has no changes, skipping...")
|
|
|
|
|
|
|
|
|
|
def _check_year_in_repo(self, repo_path: Path, year: int) -> bool:
|
|
|
|
|
"""Check if a repository contains solutions for a specific year"""
|
|
|
|
|
@@ -2336,8 +2413,12 @@ class AOCSync:
|
|
|
|
|
except Exception as e:
|
|
|
|
|
logger.error(f"Error building Podman image: {e}")
|
|
|
|
|
|
|
|
|
|
def sync_all(self):
|
|
|
|
|
def sync_all(self, force: bool = None):
|
|
|
|
|
"""Sync all repositories"""
|
|
|
|
|
if force is not None:
|
|
|
|
|
original_force = self.force_rerun
|
|
|
|
|
self.force_rerun = force
|
|
|
|
|
|
|
|
|
|
logger.info("Starting sync of all repositories...")
|
|
|
|
|
|
|
|
|
|
# Clear log file at start of sync
|
|
|
|
|
@@ -2363,6 +2444,102 @@ class AOCSync:
|
|
|
|
|
# Rsync output if configured
|
|
|
|
|
self._rsync_output()
|
|
|
|
|
|
|
|
|
|
if force is not None:
|
|
|
|
|
self.force_rerun = original_force
|
|
|
|
|
|
|
|
|
|
def sync_repo(self, repo_name: str, force: bool = True):
|
|
|
|
|
"""Sync a specific repository by name"""
|
|
|
|
|
logger.info(f"Starting sync for repository: {repo_name} (force={force})...")
|
|
|
|
|
|
|
|
|
|
original_force = self.force_rerun
|
|
|
|
|
self.force_rerun = force
|
|
|
|
|
|
|
|
|
|
# Append to log file instead of clearing
|
|
|
|
|
log_file = Path(self.config.output_dir) / 'cargo-aoc.log'
|
|
|
|
|
log_file.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
with open(log_file, 'a', encoding='utf-8') as f:
|
|
|
|
|
f.write(f"\n{'#'*80}\n")
|
|
|
|
|
f.write(f"# Sync started for {repo_name} at {datetime.now().isoformat()}\n")
|
|
|
|
|
f.write(f"{'#'*80}\n\n")
|
|
|
|
|
|
|
|
|
|
found = False
|
|
|
|
|
for repo_config in self.config.repositories:
|
|
|
|
|
if repo_config['name'] == repo_name:
|
|
|
|
|
found = True
|
|
|
|
|
try:
|
|
|
|
|
self.process_repository(repo_config, repo_name)
|
|
|
|
|
except Exception as e:
|
|
|
|
|
logger.error(f"Error processing repository {repo_name}: {e}")
|
|
|
|
|
break
|
|
|
|
|
|
|
|
|
|
if not found:
|
|
|
|
|
logger.error(f"Repository {repo_name} not found")
|
|
|
|
|
|
|
|
|
|
# Generate HTML report
|
|
|
|
|
logger.info("Generating HTML report...")
|
|
|
|
|
self.html_gen.generate(self.db, self.config)
|
|
|
|
|
|
|
|
|
|
# Rsync output if configured
|
|
|
|
|
self._rsync_output()
|
|
|
|
|
|
|
|
|
|
self.force_rerun = original_force
|
|
|
|
|
|
|
|
|
|
def sync_year(self, year: int, force: bool = True):
|
|
|
|
|
"""Sync all repositories for a specific year"""
|
|
|
|
|
logger.info(f"Starting sync for year: {year} (force={force})...")
|
|
|
|
|
|
|
|
|
|
original_force = self.force_rerun
|
|
|
|
|
self.force_rerun = force
|
|
|
|
|
|
|
|
|
|
# Append to log file instead of clearing
|
|
|
|
|
log_file = Path(self.config.output_dir) / 'cargo-aoc.log'
|
|
|
|
|
log_file.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
with open(log_file, 'a', encoding='utf-8') as f:
|
|
|
|
|
f.write(f"\n{'#'*80}\n")
|
|
|
|
|
f.write(f"# Sync started for year {year} at {datetime.now().isoformat()}\n")
|
|
|
|
|
f.write(f"{'#'*80}\n\n")
|
|
|
|
|
|
|
|
|
|
for repo_config in self.config.repositories:
|
|
|
|
|
user_name = repo_config['name']
|
|
|
|
|
repo_type = repo_config.get('type', 'single')
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
if repo_type == 'single':
|
|
|
|
|
# Check if this repo has the year
|
|
|
|
|
url = repo_config['url']
|
|
|
|
|
local_path = repo_config['local_path']
|
|
|
|
|
if self.git_manager.clone_or_update_repo(url, local_path):
|
|
|
|
|
repo_path = Path(local_path)
|
|
|
|
|
config_years = repo_config.get('years', [])
|
|
|
|
|
years_to_process = config_years if config_years else CargoAOCRunner.extract_years_from_repo(repo_path)
|
|
|
|
|
|
|
|
|
|
if year in years_to_process:
|
|
|
|
|
logger.info(f"Processing {user_name} year {year}...")
|
|
|
|
|
self._run_and_store_benchmarks(repo_path, year, user_name,
|
|
|
|
|
repo_url=url, is_multi_year=False)
|
|
|
|
|
|
|
|
|
|
elif repo_type == 'multi-year':
|
|
|
|
|
years_config = repo_config.get('years', [])
|
|
|
|
|
for year_config in years_config:
|
|
|
|
|
if year_config['year'] == year:
|
|
|
|
|
url = year_config['url']
|
|
|
|
|
local_path = year_config['local_path']
|
|
|
|
|
if self.git_manager.clone_or_update_repo(url, local_path):
|
|
|
|
|
repo_path = Path(local_path)
|
|
|
|
|
self._run_and_store_benchmarks(repo_path, year, user_name,
|
|
|
|
|
repo_url=url, is_multi_year=True)
|
|
|
|
|
except Exception as e:
|
|
|
|
|
logger.error(f"Error processing repository {user_name} for year {year}: {e}")
|
|
|
|
|
|
|
|
|
|
# Generate HTML report
|
|
|
|
|
logger.info("Generating HTML report...")
|
|
|
|
|
self.html_gen.generate(self.db, self.config)
|
|
|
|
|
|
|
|
|
|
# Rsync output if configured
|
|
|
|
|
self._rsync_output()
|
|
|
|
|
|
|
|
|
|
self.force_rerun = original_force
|
|
|
|
|
|
|
|
|
|
def _rsync_output(self):
|
|
|
|
|
"""Rsync output directory to remote server if configured"""
|
|
|
|
|
rsync_config = self.config.rsync_config
|
|
|
|
|
@@ -2417,6 +2594,356 @@ class AOCSync:
|
|
|
|
|
logger.info("Stopped by user")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class WebServer:
|
|
|
|
|
"""Simple web server for viewing logs and triggering refreshes"""
|
|
|
|
|
|
|
|
|
|
def __init__(self, sync: AOCSync, host: str = '0.0.0.0', port: int = 8080):
|
|
|
|
|
self.sync = sync
|
|
|
|
|
self.host = host
|
|
|
|
|
self.port = port
|
|
|
|
|
self.app = None
|
|
|
|
|
self._setup_app()
|
|
|
|
|
|
|
|
|
|
def _setup_app(self):
|
|
|
|
|
"""Setup Flask application"""
|
|
|
|
|
if not FLASK_AVAILABLE:
|
|
|
|
|
raise ImportError("Flask is required for web server. Install with: pip install Flask")
|
|
|
|
|
self.app = Flask(__name__)
|
|
|
|
|
|
|
|
|
|
@self.app.route('/')
|
|
|
|
|
def index():
|
|
|
|
|
return self._get_index_page()
|
|
|
|
|
|
|
|
|
|
@self.app.route('/logs')
|
|
|
|
|
def logs():
|
|
|
|
|
"""View logs"""
|
|
|
|
|
log_file = Path(self.sync.config.output_dir) / 'cargo-aoc.log'
|
|
|
|
|
if log_file.exists():
|
|
|
|
|
with open(log_file, 'r', encoding='utf-8') as f:
|
|
|
|
|
content = f.read()
|
|
|
|
|
return Response(content, mimetype='text/plain')
|
|
|
|
|
return "No log file found", 404
|
|
|
|
|
|
|
|
|
|
@self.app.route('/api/refresh/all', methods=['POST'])
|
|
|
|
|
def refresh_all():
|
|
|
|
|
"""Trigger refresh for all repositories"""
|
|
|
|
|
thread = threading.Thread(target=self.sync.sync_all, kwargs={'force': True})
|
|
|
|
|
thread.daemon = True
|
|
|
|
|
thread.start()
|
|
|
|
|
return jsonify({'status': 'started', 'message': 'Refresh started for all repositories'})
|
|
|
|
|
|
|
|
|
|
@self.app.route('/api/sync/normal', methods=['POST'])
|
|
|
|
|
def sync_normal():
|
|
|
|
|
"""Trigger normal sync loop (without force)"""
|
|
|
|
|
thread = threading.Thread(target=self.sync.sync_all, kwargs={'force': False})
|
|
|
|
|
thread.daemon = True
|
|
|
|
|
thread.start()
|
|
|
|
|
return jsonify({'status': 'started', 'message': 'Normal sync started (will skip unchanged repositories)'})
|
|
|
|
|
|
|
|
|
|
@self.app.route('/api/refresh/repo/<repo_name>', methods=['POST'])
|
|
|
|
|
def refresh_repo(repo_name):
|
|
|
|
|
"""Trigger refresh for a specific repository"""
|
|
|
|
|
thread = threading.Thread(target=self.sync.sync_repo, args=(repo_name,), kwargs={'force': True})
|
|
|
|
|
thread.daemon = True
|
|
|
|
|
thread.start()
|
|
|
|
|
return jsonify({'status': 'started', 'message': f'Refresh started for repository: {repo_name}'})
|
|
|
|
|
|
|
|
|
|
@self.app.route('/api/refresh/year/<int:year>', methods=['POST'])
|
|
|
|
|
def refresh_year(year):
|
|
|
|
|
"""Trigger refresh for a specific year"""
|
|
|
|
|
thread = threading.Thread(target=self.sync.sync_year, args=(year,), kwargs={'force': True})
|
|
|
|
|
thread.daemon = True
|
|
|
|
|
thread.start()
|
|
|
|
|
return jsonify({'status': 'started', 'message': f'Refresh started for year: {year}'})
|
|
|
|
|
|
|
|
|
|
@self.app.route('/api/repos', methods=['GET'])
|
|
|
|
|
def get_repos():
|
|
|
|
|
"""Get list of repositories"""
|
|
|
|
|
repos = [{'name': r['name'], 'type': r.get('type', 'single')} for r in self.sync.config.repositories]
|
|
|
|
|
return jsonify({'repos': repos})
|
|
|
|
|
|
|
|
|
|
@self.app.route('/api/years', methods=['GET'])
|
|
|
|
|
def get_years():
|
|
|
|
|
"""Get list of years"""
|
|
|
|
|
years = self.sync.db.get_all_years()
|
|
|
|
|
return jsonify({'years': sorted(years, reverse=True)})
|
|
|
|
|
|
|
|
|
|
def _get_index_page(self):
|
|
|
|
|
"""Generate the main web interface page"""
|
|
|
|
|
repos = [r['name'] for r in self.sync.config.repositories]
|
|
|
|
|
years = sorted(self.sync.db.get_all_years(), reverse=True)
|
|
|
|
|
|
|
|
|
|
# Build HTML - use regular string and format variables manually to avoid brace escaping issues
|
|
|
|
|
html = """<!DOCTYPE html>
|
|
|
|
|
<html lang="en">
|
|
|
|
|
<head>
|
|
|
|
|
<meta charset="UTF-8">
|
|
|
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
|
|
|
<title>AOC Sync Control Panel</title>
|
|
|
|
|
<style>
|
|
|
|
|
* {
|
|
|
|
|
margin: 0;
|
|
|
|
|
padding: 0;
|
|
|
|
|
box-sizing: border-box;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
body {
|
|
|
|
|
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif;
|
|
|
|
|
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
|
|
|
|
min-height: 100vh;
|
|
|
|
|
padding: 20px;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
.container {
|
|
|
|
|
max-width: 1200px;
|
|
|
|
|
margin: 0 auto;
|
|
|
|
|
background: white;
|
|
|
|
|
border-radius: 8px;
|
|
|
|
|
box-shadow: 0 10px 30px rgba(0,0,0,0.2);
|
|
|
|
|
padding: 30px;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
h1 {
|
|
|
|
|
color: #333;
|
|
|
|
|
margin-bottom: 10px;
|
|
|
|
|
font-size: 2em;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
.section {
|
|
|
|
|
margin-bottom: 30px;
|
|
|
|
|
padding: 20px;
|
|
|
|
|
background: #f8f9fa;
|
|
|
|
|
border-radius: 6px;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
.section h2 {
|
|
|
|
|
color: #667eea;
|
|
|
|
|
margin-bottom: 15px;
|
|
|
|
|
font-size: 1.3em;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
.button-group {
|
|
|
|
|
display: flex;
|
|
|
|
|
flex-wrap: wrap;
|
|
|
|
|
gap: 10px;
|
|
|
|
|
margin-bottom: 15px;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
button {
|
|
|
|
|
padding: 10px 20px;
|
|
|
|
|
background: #667eea;
|
|
|
|
|
color: white;
|
|
|
|
|
border: none;
|
|
|
|
|
border-radius: 4px;
|
|
|
|
|
cursor: pointer;
|
|
|
|
|
font-size: 0.9em;
|
|
|
|
|
transition: background 0.2s;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
button:hover {
|
|
|
|
|
background: #5568d3;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
button:disabled {
|
|
|
|
|
background: #ccc;
|
|
|
|
|
cursor: not-allowed;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
.button-danger {
|
|
|
|
|
background: #dc3545;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
.button-danger:hover {
|
|
|
|
|
background: #c82333;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
.status {
|
|
|
|
|
margin-top: 10px;
|
|
|
|
|
padding: 10px;
|
|
|
|
|
border-radius: 4px;
|
|
|
|
|
display: none;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
.status.success {
|
|
|
|
|
background: #d4edda;
|
|
|
|
|
color: #155724;
|
|
|
|
|
border: 1px solid #c3e6cb;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
.status.error {
|
|
|
|
|
background: #f8d7da;
|
|
|
|
|
color: #721c24;
|
|
|
|
|
border: 1px solid #f5c6cb;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
.log-viewer {
|
|
|
|
|
background: #1e1e1e;
|
|
|
|
|
color: #d4d4d4;
|
|
|
|
|
padding: 15px;
|
|
|
|
|
border-radius: 4px;
|
|
|
|
|
font-family: 'Courier New', monospace;
|
|
|
|
|
font-size: 0.85em;
|
|
|
|
|
max-height: 500px;
|
|
|
|
|
overflow-y: auto;
|
|
|
|
|
white-space: pre-wrap;
|
|
|
|
|
word-wrap: break-word;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
.log-viewer a {
|
|
|
|
|
color: #4ec9b0;
|
|
|
|
|
text-decoration: none;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
.log-viewer a:hover {
|
|
|
|
|
text-decoration: underline;
|
|
|
|
|
}
|
|
|
|
|
</style>
|
|
|
|
|
</head>
|
|
|
|
|
<body>
|
|
|
|
|
<div class="container">
|
|
|
|
|
<h1>🎄 AOC Sync Control Panel</h1>
|
|
|
|
|
|
|
|
|
|
<div class="section">
|
|
|
|
|
<h2>Refresh Controls</h2>
|
|
|
|
|
|
|
|
|
|
<div style="margin-bottom: 20px;">
|
|
|
|
|
<h3 style="margin-bottom: 10px; color: #555;">Sync All</h3>
|
|
|
|
|
<button onclick="syncNormal()" id="btn-sync-normal" style="background: #28a745; margin-right: 10px;">▶️ Normal Sync (Skip Unchanged)</button>
|
|
|
|
|
<button onclick="refreshAll()" id="btn-refresh-all">🔄 Force Refresh All Repositories</button>
|
|
|
|
|
<div class="status" id="status-all"></div>
|
|
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
<div style="margin-bottom: 20px;">
|
|
|
|
|
<h3 style="margin-bottom: 10px; color: #555;">Refresh by Repository</h3>
|
|
|
|
|
<div class="button-group">
|
|
|
|
|
"""
|
|
|
|
|
for repo in repos:
|
|
|
|
|
# Use double quotes for outer string, single quotes for JavaScript
|
|
|
|
|
html += f" <button onclick=\"refreshRepo('{repo}')\" id=\"btn-repo-{repo}\">🔄 {repo}</button>\n"
|
|
|
|
|
|
|
|
|
|
html += """ </div>
|
|
|
|
|
<div class="status" id="status-repo"></div>
|
|
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
<div style="margin-bottom: 20px;">
|
|
|
|
|
<h3 style="margin-bottom: 10px; color: #555;">Refresh by Year</h3>
|
|
|
|
|
<div class="button-group">
|
|
|
|
|
"""
|
|
|
|
|
for year in years:
|
|
|
|
|
html += f' <button onclick="refreshYear({year})" id="btn-year-{year}">🔄 {year}</button>\n'
|
|
|
|
|
|
|
|
|
|
html += """ </div>
|
|
|
|
|
<div class="status" id="status-year"></div>
|
|
|
|
|
</div>
|
|
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
<div class="section">
|
|
|
|
|
<h2>Logs</h2>
|
|
|
|
|
<p style="margin-bottom: 10px; color: #666;">
|
|
|
|
|
<a href="/logs" target="_blank" style="color: #667eea;">📋 View Full Logs</a>
|
|
|
|
|
</p>
|
|
|
|
|
<div class="log-viewer" id="log-viewer">Loading logs...</div>
|
|
|
|
|
</div>
|
|
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
<script>
|
|
|
|
|
function showStatus(elementId, message, isError = false) {
|
|
|
|
|
const status = document.getElementById(elementId);
|
|
|
|
|
status.textContent = message;
|
|
|
|
|
status.className = 'status ' + (isError ? 'error' : 'success');
|
|
|
|
|
status.style.display = 'block';
|
|
|
|
|
setTimeout(() => {
|
|
|
|
|
status.style.display = 'none';
|
|
|
|
|
}, 5000);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function syncNormal() {
|
|
|
|
|
const btn = document.getElementById('btn-sync-normal');
|
|
|
|
|
btn.disabled = true;
|
|
|
|
|
try {
|
|
|
|
|
const response = await fetch('/api/sync/normal', { method: 'POST' });
|
|
|
|
|
const data = await response.json();
|
|
|
|
|
showStatus('status-all', data.message || 'Normal sync started', false);
|
|
|
|
|
} catch (error) {
|
|
|
|
|
showStatus('status-all', 'Error: ' + error.message, true);
|
|
|
|
|
} finally {
|
|
|
|
|
btn.disabled = false;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function refreshAll() {
|
|
|
|
|
const btn = document.getElementById('btn-refresh-all');
|
|
|
|
|
btn.disabled = true;
|
|
|
|
|
try {
|
|
|
|
|
const response = await fetch('/api/refresh/all', { method: 'POST' });
|
|
|
|
|
const data = await response.json();
|
|
|
|
|
showStatus('status-all', data.message || 'Refresh started', false);
|
|
|
|
|
} catch (error) {
|
|
|
|
|
showStatus('status-all', 'Error: ' + error.message, true);
|
|
|
|
|
} finally {
|
|
|
|
|
btn.disabled = false;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function refreshRepo(repoName) {
|
|
|
|
|
const btn = document.getElementById('btn-repo-' + repoName);
|
|
|
|
|
btn.disabled = true;
|
|
|
|
|
try {
|
|
|
|
|
const response = await fetch('/api/refresh/repo/' + encodeURIComponent(repoName), { method: 'POST' });
|
|
|
|
|
const data = await response.json();
|
|
|
|
|
showStatus('status-repo', data.message || 'Refresh started', false);
|
|
|
|
|
} catch (error) {
|
|
|
|
|
showStatus('status-repo', 'Error: ' + error.message, true);
|
|
|
|
|
} finally {
|
|
|
|
|
btn.disabled = false;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function refreshYear(year) {
|
|
|
|
|
const btn = document.getElementById('btn-year-' + year);
|
|
|
|
|
btn.disabled = true;
|
|
|
|
|
try {
|
|
|
|
|
const response = await fetch('/api/refresh/year/' + year, { method: 'POST' });
|
|
|
|
|
const data = await response.json();
|
|
|
|
|
showStatus('status-year', data.message || 'Refresh started', false);
|
|
|
|
|
} catch (error) {
|
|
|
|
|
showStatus('status-year', 'Error: ' + error.message, true);
|
|
|
|
|
} finally {
|
|
|
|
|
btn.disabled = false;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function loadLogs() {
|
|
|
|
|
try {
|
|
|
|
|
const response = await fetch('/logs');
|
|
|
|
|
if (response.ok) {
|
|
|
|
|
const text = await response.text();
|
|
|
|
|
const logViewer = document.getElementById('log-viewer');
|
|
|
|
|
// Show last 5000 characters
|
|
|
|
|
logViewer.textContent = text.slice(-5000);
|
|
|
|
|
logViewer.scrollTop = logViewer.scrollHeight;
|
|
|
|
|
} else {
|
|
|
|
|
document.getElementById('log-viewer').textContent = 'No logs available';
|
|
|
|
|
}
|
|
|
|
|
} catch (error) {
|
|
|
|
|
document.getElementById('log-viewer').textContent = 'Error loading logs: ' + error.message;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Load logs on page load and refresh every 5 seconds
|
|
|
|
|
loadLogs();
|
|
|
|
|
setInterval(loadLogs, 5000);
|
|
|
|
|
</script>
|
|
|
|
|
</body>
|
|
|
|
|
</html>"""
|
|
|
|
|
return html
|
|
|
|
|
|
|
|
|
|
def run(self):
|
|
|
|
|
"""Run the web server"""
|
|
|
|
|
logger.info(f"Starting web server on http://{self.host}:{self.port}")
|
|
|
|
|
self.app.run(host=self.host, port=self.port, debug=False, use_reloader=False)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def main():
|
|
|
|
|
"""Main entry point"""
|
|
|
|
|
import argparse
|
|
|
|
|
@@ -2426,15 +2953,35 @@ def main():
|
|
|
|
|
parser.add_argument('--once', action='store_true', help='Run once instead of continuously')
|
|
|
|
|
parser.add_argument('--force', '--rerun-all', action='store_true', dest='force_rerun',
|
|
|
|
|
help='Force rerun all days even if repository has not changed')
|
|
|
|
|
parser.add_argument('--web', action='store_true', help='Start web server for logs and refresh controls')
|
|
|
|
|
parser.add_argument('--web-host', default='0.0.0.0', help='Web server host (default: 0.0.0.0)')
|
|
|
|
|
parser.add_argument('--web-port', type=int, default=8080, help='Web server port (default: 8080)')
|
|
|
|
|
|
|
|
|
|
args = parser.parse_args()
|
|
|
|
|
|
|
|
|
|
sync = AOCSync(args.config, force_rerun=args.force_rerun)
|
|
|
|
|
|
|
|
|
|
if args.once:
|
|
|
|
|
sync.sync_all()
|
|
|
|
|
if args.web:
|
|
|
|
|
if not FLASK_AVAILABLE:
|
|
|
|
|
logger.error("Flask is required for web server. Install with: pip install Flask")
|
|
|
|
|
sys.exit(1)
|
|
|
|
|
# Start web server
|
|
|
|
|
web_server = WebServer(sync, host=args.web_host, port=args.web_port)
|
|
|
|
|
if args.once:
|
|
|
|
|
# Run once then start web server
|
|
|
|
|
sync.sync_all()
|
|
|
|
|
web_server.run()
|
|
|
|
|
else:
|
|
|
|
|
# Start web server in background thread, then run continuous sync
|
|
|
|
|
web_thread = threading.Thread(target=web_server.run)
|
|
|
|
|
web_thread.daemon = True
|
|
|
|
|
web_thread.start()
|
|
|
|
|
sync.run_continuous()
|
|
|
|
|
else:
|
|
|
|
|
sync.run_continuous()
|
|
|
|
|
if args.once:
|
|
|
|
|
sync.sync_all()
|
|
|
|
|
else:
|
|
|
|
|
sync.run_continuous()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
|
|