Add ability to run cargo aoc under docker
This commit is contained in:
parent
88ec3631ab
commit
492d06b0d6
146
aocsync.py
146
aocsync.py
@ -86,6 +86,18 @@ class Config:
|
|||||||
def rsync_config(self) -> Optional[dict]:
|
def rsync_config(self) -> Optional[dict]:
|
||||||
return self.config.get('rsync')
|
return self.config.get('rsync')
|
||||||
|
|
||||||
|
@property
|
||||||
|
def docker_config(self) -> dict:
|
||||||
|
"""Get Docker configuration with defaults"""
|
||||||
|
docker_config = self.config.get('docker', {})
|
||||||
|
return {
|
||||||
|
'build_cache_dir': docker_config.get('build_cache_dir', ''),
|
||||||
|
'registry_cache_dir': docker_config.get('registry_cache_dir', ''),
|
||||||
|
'memory': docker_config.get('memory', '2g'),
|
||||||
|
'cpus': docker_config.get('cpus', '2'),
|
||||||
|
'image': docker_config.get('image', 'rust:latest')
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
class Database:
|
class Database:
|
||||||
"""SQLite database for storing performance results"""
|
"""SQLite database for storing performance results"""
|
||||||
@ -560,10 +572,107 @@ class CargoAOCRunner:
|
|||||||
logger.warning(f"Could not get recent commits for {repo_path}: {e}")
|
logger.warning(f"Could not get recent commits for {repo_path}: {e}")
|
||||||
return commits
|
return commits
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _run_cargo_aoc_in_container(work_dir: Path, day: int, repo_root: Path, docker_config: dict) -> subprocess.CompletedProcess:
|
||||||
|
"""Run cargo aoc in a Docker container for security
|
||||||
|
|
||||||
|
Args:
|
||||||
|
work_dir: Working directory (year directory) - can be absolute or relative
|
||||||
|
day: Day number to run
|
||||||
|
repo_root: Absolute path to repository root
|
||||||
|
docker_config: Docker configuration dictionary
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
CompletedProcess with stdout, stderr, returncode
|
||||||
|
"""
|
||||||
|
repo_root = Path(repo_root).resolve()
|
||||||
|
work_dir = Path(work_dir).resolve()
|
||||||
|
|
||||||
|
# Ensure work_dir is under repo_root
|
||||||
|
try:
|
||||||
|
work_dir_rel = str(work_dir.relative_to(repo_root))
|
||||||
|
except ValueError:
|
||||||
|
# If work_dir is not under repo_root, this is an error
|
||||||
|
raise ValueError(f"work_dir {work_dir} is not under repo_root {repo_root}")
|
||||||
|
|
||||||
|
# Determine build cache directory
|
||||||
|
build_cache_dir = docker_config.get('build_cache_dir', '')
|
||||||
|
use_temp_build = False
|
||||||
|
temp_build_dir = None
|
||||||
|
|
||||||
|
if build_cache_dir:
|
||||||
|
# Use persistent build cache directory
|
||||||
|
build_cache_path = Path(build_cache_dir).resolve()
|
||||||
|
build_cache_path.mkdir(parents=True, exist_ok=True)
|
||||||
|
logger.info(f"Using persistent build cache: {build_cache_path}")
|
||||||
|
else:
|
||||||
|
# Create a temporary directory for cargo build artifacts (outside repo)
|
||||||
|
import tempfile
|
||||||
|
temp_build_dir = tempfile.mkdtemp(prefix='cargo-aoc-build-')
|
||||||
|
build_cache_path = Path(temp_build_dir)
|
||||||
|
use_temp_build = True
|
||||||
|
logger.info(f"Using temporary build cache: {build_cache_path}")
|
||||||
|
|
||||||
|
# Determine registry cache directory
|
||||||
|
registry_cache_dir = docker_config.get('registry_cache_dir', '')
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Build Docker command
|
||||||
|
docker_cmd = [
|
||||||
|
'docker', 'run',
|
||||||
|
'--rm', # Remove container after execution
|
||||||
|
'--network=none', # No network access
|
||||||
|
'--memory', docker_config.get('memory', '2g'), # Limit memory
|
||||||
|
'--cpus', str(docker_config.get('cpus', '2')), # Limit CPU
|
||||||
|
'--read-only', # Read-only root filesystem
|
||||||
|
'--tmpfs', '/tmp:rw,noexec,nosuid,size=1g', # Writable /tmp for cargo
|
||||||
|
'-v', f'{repo_root}:/repo:ro', # Mount repo read-only
|
||||||
|
'-v', f'{build_cache_path}:/build:rw', # Writable build directory
|
||||||
|
'-w', f'/repo/{work_dir_rel}', # Working directory in container
|
||||||
|
]
|
||||||
|
|
||||||
|
# Handle cargo registry cache
|
||||||
|
if registry_cache_dir:
|
||||||
|
# Use persistent registry cache
|
||||||
|
registry_cache_path = Path(registry_cache_dir).resolve()
|
||||||
|
registry_cache_path.mkdir(parents=True, exist_ok=True)
|
||||||
|
docker_cmd.extend(['-v', f'{registry_cache_path}:/root/.cargo/registry:rw'])
|
||||||
|
logger.info(f"Using persistent registry cache: {registry_cache_path}")
|
||||||
|
else:
|
||||||
|
# Use tmpfs for registry cache (cleared after each run)
|
||||||
|
docker_cmd.extend(['--tmpfs', '/root/.cargo/registry:rw,noexec,nosuid,size=100m'])
|
||||||
|
|
||||||
|
# Add Docker image and command
|
||||||
|
docker_cmd.extend([
|
||||||
|
docker_config.get('image', 'rust:latest'),
|
||||||
|
'cargo', 'aoc', '--day', str(day)
|
||||||
|
])
|
||||||
|
|
||||||
|
# Set CARGO_TARGET_DIR to use the mounted build directory
|
||||||
|
env = os.environ.copy()
|
||||||
|
env['CARGO_TARGET_DIR'] = '/build/target'
|
||||||
|
|
||||||
|
result = subprocess.run(
|
||||||
|
docker_cmd,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=300, # 5 minute timeout
|
||||||
|
env=env
|
||||||
|
)
|
||||||
|
|
||||||
|
return result
|
||||||
|
finally:
|
||||||
|
# Clean up temporary build directory if we created one
|
||||||
|
if use_temp_build and temp_build_dir:
|
||||||
|
try:
|
||||||
|
shutil.rmtree(temp_build_dir)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Failed to clean up temp build directory {temp_build_dir}: {e}")
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def run_benchmarks(repo_path: Path, year: int, user: str = "unknown",
|
def run_benchmarks(repo_path: Path, year: int, user: str = "unknown",
|
||||||
repo_url: str = "", is_multi_year: bool = False,
|
repo_url: str = "", is_multi_year: bool = False,
|
||||||
log_file: Optional[Path] = None) -> List[PerformanceResult]:
|
log_file: Optional[Path] = None, docker_config: Optional[dict] = None) -> List[PerformanceResult]:
|
||||||
"""Run cargo aoc benchmarks and parse results
|
"""Run cargo aoc benchmarks and parse results
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@ -575,17 +684,19 @@ class CargoAOCRunner:
|
|||||||
log_file: Optional path to log file to append cargo aoc output to
|
log_file: Optional path to log file to append cargo aoc output to
|
||||||
"""
|
"""
|
||||||
results = []
|
results = []
|
||||||
repo_path = Path(repo_path)
|
repo_path = Path(repo_path).resolve()
|
||||||
|
|
||||||
# Get git revision
|
# Get git revision
|
||||||
git_rev = CargoAOCRunner.get_git_rev(repo_path)
|
git_rev = CargoAOCRunner.get_git_rev(repo_path)
|
||||||
|
|
||||||
# Determine the working directory
|
# Determine the working directory and repo root
|
||||||
if is_multi_year:
|
if is_multi_year:
|
||||||
# For multi-year repos, repo_path is already the year directory
|
# For multi-year repos, repo_path is already the year directory
|
||||||
work_dir = repo_path
|
work_dir = repo_path
|
||||||
|
repo_root = repo_path # For multi-year, repo_path is the repo root
|
||||||
else:
|
else:
|
||||||
# For single repos, check if we need to navigate to a year subdirectory
|
# For single repos, check if we need to navigate to a year subdirectory
|
||||||
|
repo_root = repo_path # Repo root is the repo_path
|
||||||
work_dir = repo_path
|
work_dir = repo_path
|
||||||
year_dir = repo_path / str(year)
|
year_dir = repo_path / str(year)
|
||||||
if year_dir.exists() and year_dir.is_dir():
|
if year_dir.exists() and year_dir.is_dir():
|
||||||
@ -603,17 +714,18 @@ class CargoAOCRunner:
|
|||||||
|
|
||||||
for day in days:
|
for day in days:
|
||||||
try:
|
try:
|
||||||
logger.info(f"Running cargo aoc for {user} year {year} day {day} in {work_dir}")
|
logger.info(f"Running cargo aoc for {user} year {year} day {day} in {work_dir} (in Docker container)")
|
||||||
# Run cargo aoc for this day (no year flag, must be in correct directory)
|
# Run cargo aoc in a Docker container for security
|
||||||
cmd = ['cargo', 'aoc', '--day', str(day)]
|
# Use default docker_config if not provided
|
||||||
|
if docker_config is None:
|
||||||
result = subprocess.run(
|
docker_config = {
|
||||||
cmd,
|
'build_cache_dir': '',
|
||||||
cwd=work_dir,
|
'registry_cache_dir': '',
|
||||||
capture_output=True,
|
'memory': '2g',
|
||||||
text=True,
|
'cpus': '2',
|
||||||
timeout=300 # 5 minute timeout per day
|
'image': 'rust:latest'
|
||||||
)
|
}
|
||||||
|
result = CargoAOCRunner._run_cargo_aoc_in_container(work_dir, day, repo_root, docker_config)
|
||||||
|
|
||||||
# Write to log file if provided
|
# Write to log file if provided
|
||||||
if log_file:
|
if log_file:
|
||||||
@ -625,7 +737,7 @@ class CargoAOCRunner:
|
|||||||
with open(log_file, 'a', encoding='utf-8') as f:
|
with open(log_file, 'a', encoding='utf-8') as f:
|
||||||
f.write(f"\n{'='*80}\n")
|
f.write(f"\n{'='*80}\n")
|
||||||
f.write(f"[{timestamp}] {user} - Year {year} - Day {day}\n")
|
f.write(f"[{timestamp}] {user} - Year {year} - Day {day}\n")
|
||||||
f.write(f"Command: {' '.join(cmd)}\n")
|
f.write(f"Command: cargo aoc --day {day} (in Docker container)\n")
|
||||||
f.write(f"Working Directory: {work_dir}\n")
|
f.write(f"Working Directory: {work_dir}\n")
|
||||||
f.write(f"Return Code: {result.returncode}\n")
|
f.write(f"Return Code: {result.returncode}\n")
|
||||||
f.write(f"{'='*80}\n")
|
f.write(f"{'='*80}\n")
|
||||||
@ -1956,9 +2068,11 @@ class AOCSync:
|
|||||||
# Create log file path in output directory
|
# Create log file path in output directory
|
||||||
log_file = Path(self.config.output_dir) / 'cargo-aoc.log'
|
log_file = Path(self.config.output_dir) / 'cargo-aoc.log'
|
||||||
log_file.parent.mkdir(parents=True, exist_ok=True)
|
log_file.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
# Get Docker configuration
|
||||||
|
docker_config = self.config.docker_config
|
||||||
results = CargoAOCRunner.run_benchmarks(repo_path, year=year, user=user,
|
results = CargoAOCRunner.run_benchmarks(repo_path, year=year, user=user,
|
||||||
repo_url=repo_url, is_multi_year=is_multi_year,
|
repo_url=repo_url, is_multi_year=is_multi_year,
|
||||||
log_file=log_file)
|
log_file=log_file, docker_config=docker_config)
|
||||||
|
|
||||||
# Store results
|
# Store results
|
||||||
for result in results:
|
for result in results:
|
||||||
|
|||||||
17
config.yaml
17
config.yaml
@ -13,6 +13,23 @@ rsync:
|
|||||||
enabled: true
|
enabled: true
|
||||||
destination: "xinu.tv:/var/www/static/aoc/"
|
destination: "xinu.tv:/var/www/static/aoc/"
|
||||||
|
|
||||||
|
# Docker container configuration for running cargo aoc
|
||||||
|
docker:
|
||||||
|
# Persistent directory for cargo build artifacts (speeds up rebuilds)
|
||||||
|
# If not specified, uses temporary directory that's cleaned up after each run
|
||||||
|
build_cache_dir: "/tmp/aocsync/build_cache_dir"
|
||||||
|
|
||||||
|
# Persistent directory for cargo registry cache (downloaded dependencies)
|
||||||
|
# If not specified, uses tmpfs that's cleared after each run
|
||||||
|
registry_cache_dir: "/tmp/aocsync/registry_cache_dir"
|
||||||
|
|
||||||
|
# Container resource limits
|
||||||
|
memory: "2g" # Memory limit
|
||||||
|
cpus: "2" # CPU limit
|
||||||
|
|
||||||
|
# Docker image to use
|
||||||
|
image: "rust:latest"
|
||||||
|
|
||||||
# Repositories to monitor
|
# Repositories to monitor
|
||||||
repositories:
|
repositories:
|
||||||
# Example: Single repository with all years
|
# Example: Single repository with all years
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user