From 1c16f6547600a982184037518d1d8c0fb2e7bb69 Mon Sep 17 00:00:00 2001 From: David Culbreth Date: Fri, 20 Mar 2026 19:50:44 -0500 Subject: [PATCH] addressing configuration dependency issues --- Makefile | 8 +- docker-compose.yaml | 28 ++- docker/DOCKER_BUILD_RACE_CONDITIONS.md | 6 +- docker/Dockerfile | 151 ------------- docker/Dockerfile.optimized | 7 +- docker/Dockerfile.sensor.optimized | 10 +- docker/Dockerfile.worker | 300 ------------------------- docker/Dockerfile.worker.optimized | 20 +- docker/README.md | 17 +- docker/README.worker.md | 6 +- docker/enable-buildkit.sh | 2 +- 11 files changed, 56 insertions(+), 499 deletions(-) delete mode 100644 docker/Dockerfile delete mode 100644 docker/Dockerfile.worker diff --git a/Makefile b/Makefile index 985b195..84004da 100644 --- a/Makefile +++ b/Makefile @@ -233,22 +233,22 @@ docker-build-workers: docker-build-worker-base docker-build-worker-python docker docker-build-worker-base: @echo "Building base worker (shell only)..." - DOCKER_BUILDKIT=1 docker build --target worker-base -t attune-worker:base -f docker/Dockerfile.worker . + DOCKER_BUILDKIT=1 docker build --target worker-base -t attune-worker:base -f docker/Dockerfile.worker.optimized . @echo "✅ Base worker image built: attune-worker:base" docker-build-worker-python: @echo "Building Python worker (shell + python)..." - DOCKER_BUILDKIT=1 docker build --target worker-python -t attune-worker:python -f docker/Dockerfile.worker . + DOCKER_BUILDKIT=1 docker build --target worker-python -t attune-worker:python -f docker/Dockerfile.worker.optimized . @echo "✅ Python worker image built: attune-worker:python" docker-build-worker-node: @echo "Building Node.js worker (shell + node)..." - DOCKER_BUILDKIT=1 docker build --target worker-node -t attune-worker:node -f docker/Dockerfile.worker . + DOCKER_BUILDKIT=1 docker build --target worker-node -t attune-worker:node -f docker/Dockerfile.worker.optimized . @echo "✅ Node.js worker image built: attune-worker:node" docker-build-worker-full: @echo "Building full worker (all runtimes)..." - DOCKER_BUILDKIT=1 docker build --target worker-full -t attune-worker:full -f docker/Dockerfile.worker . + DOCKER_BUILDKIT=1 docker build --target worker-full -t attune-worker:full -f docker/Dockerfile.worker.optimized . @echo "✅ Full worker image built: attune-worker:full" docker-up: diff --git a/docker-compose.yaml b/docker-compose.yaml index 954c6aa..9b5918a 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -10,6 +10,10 @@ # Password: TestPass123! # See docs/testing/test-user-setup.md for custom users +# Runtime config file selection: +# ATTUNE_DOCKER_CONFIG_PATH controls the host-side config YAML mounted into services. +# Default: ./config.docker.yaml + services: # ============================================================================ # Infrastructure Services @@ -174,7 +178,7 @@ services: container_name: attune-api environment: RUST_LOG: info - ATTUNE_CONFIG: /opt/attune/config.yaml + ATTUNE_CONFIG: /opt/attune/config/config.yaml # Security - MUST set these in production via .env file ATTUNE__SECURITY__JWT_SECRET: ${JWT_SECRET:-docker-dev-secret-change-in-production} ATTUNE__SECURITY__ENCRYPTION_KEY: ${ENCRYPTION_KEY:-docker-dev-encryption-key-please-change-in-production-32plus} @@ -189,6 +193,7 @@ services: ports: - "8080:8080" volumes: + - ${ATTUNE_DOCKER_CONFIG_PATH:-./config.docker.yaml}:/opt/attune/config/config.yaml:ro - packs_data:/opt/attune/packs:rw - ./packs.dev:/opt/attune/packs.dev:rw - runtime_envs:/opt/attune/runtime_envs @@ -227,7 +232,7 @@ services: container_name: attune-executor environment: RUST_LOG: info - ATTUNE_CONFIG: /opt/attune/config.yaml + ATTUNE_CONFIG: /opt/attune/config/config.yaml ATTUNE__SECURITY__JWT_SECRET: ${JWT_SECRET:-docker-dev-secret-change-in-production} ATTUNE__SECURITY__ENCRYPTION_KEY: ${ENCRYPTION_KEY:-docker-dev-encryption-key-please-change-in-production-32plus} ATTUNE__DATABASE__URL: postgresql://attune:attune@postgres:5432/attune @@ -235,6 +240,7 @@ services: ATTUNE__CACHE__URL: redis://redis:6379 ATTUNE__WORKER__WORKER_TYPE: container volumes: + - ${ATTUNE_DOCKER_CONFIG_PATH:-./config.docker.yaml}:/opt/attune/config/config.yaml:ro - packs_data:/opt/attune/packs:ro - ./packs.dev:/opt/attune/packs.dev:rw - artifacts_data:/opt/attune/artifacts:ro @@ -276,7 +282,7 @@ services: stop_grace_period: 45s environment: RUST_LOG: info - ATTUNE_CONFIG: /opt/attune/config.yaml + ATTUNE_CONFIG: /opt/attune/config/config.yaml ATTUNE_WORKER_RUNTIMES: shell ATTUNE_WORKER_TYPE: container ATTUNE_WORKER_NAME: worker-shell-01 @@ -286,6 +292,7 @@ services: ATTUNE__MESSAGE_QUEUE__URL: amqp://attune:attune@rabbitmq:5672 ATTUNE_API_URL: http://attune-api:8080 volumes: + - ${ATTUNE_DOCKER_CONFIG_PATH:-./config.docker.yaml}:/opt/attune/config/config.yaml:ro - packs_data:/opt/attune/packs:ro - ./packs.dev:/opt/attune/packs.dev:rw - runtime_envs:/opt/attune/runtime_envs @@ -324,7 +331,7 @@ services: stop_grace_period: 45s environment: RUST_LOG: info - ATTUNE_CONFIG: /opt/attune/config.yaml + ATTUNE_CONFIG: /opt/attune/config/config.yaml ATTUNE_WORKER_RUNTIMES: shell,python ATTUNE_WORKER_TYPE: container ATTUNE_WORKER_NAME: worker-python-01 @@ -334,6 +341,7 @@ services: ATTUNE__MESSAGE_QUEUE__URL: amqp://attune:attune@rabbitmq:5672 ATTUNE_API_URL: http://attune-api:8080 volumes: + - ${ATTUNE_DOCKER_CONFIG_PATH:-./config.docker.yaml}:/opt/attune/config/config.yaml:ro - packs_data:/opt/attune/packs:ro - ./packs.dev:/opt/attune/packs.dev:rw - runtime_envs:/opt/attune/runtime_envs @@ -372,7 +380,7 @@ services: stop_grace_period: 45s environment: RUST_LOG: info - ATTUNE_CONFIG: /opt/attune/config.yaml + ATTUNE_CONFIG: /opt/attune/config/config.yaml ATTUNE_WORKER_RUNTIMES: shell,node ATTUNE_WORKER_TYPE: container ATTUNE_WORKER_NAME: worker-node-01 @@ -382,6 +390,7 @@ services: ATTUNE__MESSAGE_QUEUE__URL: amqp://attune:attune@rabbitmq:5672 ATTUNE_API_URL: http://attune-api:8080 volumes: + - ${ATTUNE_DOCKER_CONFIG_PATH:-./config.docker.yaml}:/opt/attune/config/config.yaml:ro - packs_data:/opt/attune/packs:ro - ./packs.dev:/opt/attune/packs.dev:rw - runtime_envs:/opt/attune/runtime_envs @@ -420,7 +429,7 @@ services: stop_grace_period: 45s environment: RUST_LOG: info - ATTUNE_CONFIG: /opt/attune/config.yaml + ATTUNE_CONFIG: /opt/attune/config/config.yaml ATTUNE_WORKER_RUNTIMES: shell,python,node,native ATTUNE_WORKER_TYPE: container ATTUNE_WORKER_NAME: worker-full-01 @@ -430,6 +439,7 @@ services: ATTUNE__MESSAGE_QUEUE__URL: amqp://attune:attune@rabbitmq:5672 ATTUNE_API_URL: http://attune-api:8080 volumes: + - ${ATTUNE_DOCKER_CONFIG_PATH:-./config.docker.yaml}:/opt/attune/config/config.yaml:ro - packs_data:/opt/attune/packs:ro - ./packs.dev:/opt/attune/packs.dev:rw - runtime_envs:/opt/attune/runtime_envs @@ -467,7 +477,7 @@ services: stop_grace_period: 45s environment: RUST_LOG: debug - ATTUNE_CONFIG: /opt/attune/config.yaml + ATTUNE_CONFIG: /opt/attune/config/config.yaml ATTUNE__SECURITY__JWT_SECRET: ${JWT_SECRET:-docker-dev-secret-change-in-production} ATTUNE__SECURITY__ENCRYPTION_KEY: ${ENCRYPTION_KEY:-docker-dev-encryption-key-please-change-in-production-32plus} ATTUNE__DATABASE__URL: postgresql://attune:attune@postgres:5432/attune @@ -478,6 +488,7 @@ services: ATTUNE_MQ_URL: amqp://attune:attune@rabbitmq:5672 ATTUNE_PACKS_BASE_DIR: /opt/attune/packs volumes: + - ${ATTUNE_DOCKER_CONFIG_PATH:-./config.docker.yaml}:/opt/attune/config/config.yaml:ro - packs_data:/opt/attune/packs:rw - ./packs.dev:/opt/attune/packs.dev:rw - runtime_envs:/opt/attune/runtime_envs @@ -513,7 +524,7 @@ services: container_name: attune-notifier environment: RUST_LOG: info - ATTUNE_CONFIG: /opt/attune/config.yaml + ATTUNE_CONFIG: /opt/attune/config/config.yaml ATTUNE__SECURITY__JWT_SECRET: ${JWT_SECRET:-docker-dev-secret-change-in-production} ATTUNE__SECURITY__ENCRYPTION_KEY: ${ENCRYPTION_KEY:-docker-dev-encryption-key-please-change-in-production-32plus} ATTUNE__DATABASE__URL: postgresql://attune:attune@postgres:5432/attune @@ -522,6 +533,7 @@ services: ports: - "8081:8081" volumes: + - ${ATTUNE_DOCKER_CONFIG_PATH:-./config.docker.yaml}:/opt/attune/config/config.yaml:ro - notifier_logs:/opt/attune/logs depends_on: migrations: diff --git a/docker/DOCKER_BUILD_RACE_CONDITIONS.md b/docker/DOCKER_BUILD_RACE_CONDITIONS.md index 603b29c..094b1b6 100644 --- a/docker/DOCKER_BUILD_RACE_CONDITIONS.md +++ b/docker/DOCKER_BUILD_RACE_CONDITIONS.md @@ -219,10 +219,10 @@ docker builder prune --keep-storage 10GB ### Want faster parallel builds -Remove `sharing=locked` from `docker/Dockerfile` and use cache warming: +Remove `sharing=locked` from the optimized Dockerfiles and use cache warming: ```bash -# Edit docker/Dockerfile - remove ,sharing=locked from RUN --mount lines +# Edit the optimized Dockerfiles - remove ,sharing=locked from RUN --mount lines make docker-cache-warm make docker-build ``` @@ -250,4 +250,4 @@ make docker-build **Recommended workflow**: Use `make docker-cache-warm` before `make docker-build` for faster initial builds. -**Trade-off**: Slight increase in build time (~5-10 min) for 100% reliability is worth it for production deployments. \ No newline at end of file +**Trade-off**: Slight increase in build time (~5-10 min) for 100% reliability is worth it for production deployments. diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index 904740d..0000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,151 +0,0 @@ -# Multi-stage Dockerfile for Attune Rust services -# This Dockerfile can build any of the Attune services by specifying a build argument -# Usage: DOCKER_BUILDKIT=1 docker build --build-arg SERVICE=api -f docker/Dockerfile -t attune-api . -# -# BuildKit cache mounts are used to speed up incremental builds by persisting: -# - Cargo registry and git cache (with sharing=locked to prevent race conditions) -# - Rust incremental compilation artifacts -# -# This dramatically reduces rebuild times from ~5 minutes to ~30 seconds for code-only changes. - -ARG RUST_VERSION=1.92 -ARG DEBIAN_VERSION=bookworm - -# ============================================================================ -# Stage 1: Builder - Compile the Rust services -# ============================================================================ -FROM rust:${RUST_VERSION}-${DEBIAN_VERSION} AS builder - -# Install build dependencies -RUN apt-get update && apt-get install -y \ - pkg-config \ - libssl-dev \ - ca-certificates \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /build - -# Increase rustc stack size to prevent SIGSEGV during release builds -ENV RUST_MIN_STACK=16777216 - -# Copy workspace manifests and source code -COPY Cargo.toml Cargo.lock ./ -COPY crates/ ./crates/ -COPY migrations/ ./migrations/ -COPY .sqlx/ ./.sqlx/ - -# Build argument to specify which service to build -ARG SERVICE=api - -# Build the specified service with BuildKit cache mounts -# Cache mount sharing modes prevent race conditions during parallel builds: -# - sharing=locked: Only one build can access the cache at a time (prevents file conflicts) -# - cargo registry/git: Locked to prevent "File exists" errors when extracting dependencies -# - target: Locked to prevent compilation artifact conflicts -# -# This is slower than parallel builds but eliminates race conditions. -# Alternative: Use docker-compose --build with --no-parallel flag, or build sequentially. -# -# First build: ~5-6 minutes -# Incremental builds (code changes only): ~30-60 seconds -RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ - --mount=type=cache,target=/usr/local/cargo/git,sharing=locked \ - --mount=type=cache,target=/build/target,sharing=locked \ - cargo build --release --bin attune-${SERVICE} && \ - cp /build/target/release/attune-${SERVICE} /build/attune-service-binary - -# ============================================================================ -# Stage 2: Pack Binaries Builder - Build native pack binaries with GLIBC 2.36 -# ============================================================================ -FROM rust:${RUST_VERSION}-${DEBIAN_VERSION} AS pack-builder - -# Install build dependencies -RUN apt-get update && apt-get install -y \ - pkg-config \ - libssl-dev \ - ca-certificates \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /build - -# Increase rustc stack size to prevent SIGSEGV during release builds -ENV RUST_MIN_STACK=16777216 - -# Copy workspace files -COPY Cargo.toml Cargo.lock ./ -COPY crates/ ./crates/ -COPY .sqlx/ ./.sqlx/ - -# Build pack binaries (sensors, etc.) with GLIBC 2.36 for maximum compatibility -# These binaries will work on any system with GLIBC 2.36 or newer -# IMPORTANT: Copy binaries WITHIN the cache mount, before it's unmounted -RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ - --mount=type=cache,target=/usr/local/cargo/git,sharing=locked \ - --mount=type=cache,target=/build/target,sharing=locked \ - mkdir -p /build/pack-binaries && \ - cargo build --release --bin attune-core-timer-sensor && \ - cp /build/target/release/attune-core-timer-sensor /build/pack-binaries/attune-core-timer-sensor && \ - ls -lh /build/pack-binaries/ - -# Verify binaries were copied successfully (after cache unmount) -RUN ls -lah /build/pack-binaries/ && \ - test -f /build/pack-binaries/attune-core-timer-sensor && \ - echo "Timer sensor binary built successfully" - -# ============================================================================ -# Stage 3: Runtime - Create minimal runtime image -# ============================================================================ -FROM debian:${DEBIAN_VERSION}-slim AS runtime - -# Install runtime dependencies -RUN apt-get update && apt-get install -y \ - ca-certificates \ - libssl3 \ - curl \ - && rm -rf /var/lib/apt/lists/* - -# Create non-root user -RUN useradd -m -u 1000 attune && \ - mkdir -p /opt/attune/packs /opt/attune/logs && \ - chown -R attune:attune /opt/attune - -WORKDIR /opt/attune - -# Copy the service binary from builder -# Note: We copy from /build/attune-service-binary because the cache mount is not available in COPY -COPY --from=builder /build/attune-service-binary /usr/local/bin/attune-service - -# Copy configuration for Docker Compose development -# Production: mount config files as a volume instead of baking them into the image -COPY config.docker.yaml ./config.yaml - -# Copy migrations for services that need them -COPY migrations/ ./migrations/ - -# Copy packs directory (excluding binaries that will be overwritten) -COPY packs/ ./packs/ - -# Overwrite pack binaries with ones built with compatible GLIBC from pack-builder stage -# Copy individual files to ensure they overwrite existing ones -COPY --from=pack-builder /build/pack-binaries/attune-core-timer-sensor ./packs/core/sensors/attune-core-timer-sensor - -# Make binaries executable and set ownership -RUN chmod +x ./packs/core/sensors/attune-core-timer-sensor && \ - chown -R attune:attune /opt/attune - -# Switch to non-root user -USER attune - -# Environment variables (can be overridden at runtime) -ENV RUST_LOG=info -ENV ATTUNE_CONFIG=/opt/attune/config.yaml - -# Health check (will be overridden per service in docker-compose) -HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \ - CMD curl -f http://localhost:8080/health || exit 1 - -# Expose default port (override per service) -EXPOSE 8080 - -# Run the service -CMD ["/usr/local/bin/attune-service"] diff --git a/docker/Dockerfile.optimized b/docker/Dockerfile.optimized index bbdac66..ec42e07 100644 --- a/docker/Dockerfile.optimized +++ b/docker/Dockerfile.optimized @@ -100,7 +100,7 @@ RUN apt-get update && apt-get install -y \ # Create non-root user and directories # /opt/attune/packs is mounted as a volume at runtime, not copied in RUN useradd -m -u 1000 attune && \ - mkdir -p /opt/attune/packs /opt/attune/logs /opt/attune/runtime_envs && \ + mkdir -p /opt/attune/packs /opt/attune/logs /opt/attune/runtime_envs /opt/attune/config && \ chown -R attune:attune /opt/attune WORKDIR /opt/attune @@ -109,8 +109,7 @@ WORKDIR /opt/attune # This avoids the circular dependency Docker hits when using ARG in --from paths COPY --from=builder /build/attune-service-binary /usr/local/bin/attune-service -# Copy configuration and migrations -COPY config.docker.yaml ./config.yaml +# Copy migrations only. Runtime config is mounted via Docker Compose. COPY migrations/ ./migrations/ RUN chown -R attune:attune /opt/attune @@ -118,7 +117,7 @@ RUN chown -R attune:attune /opt/attune USER attune ENV RUST_LOG=info -ENV ATTUNE_CONFIG=/opt/attune/config.yaml +ENV ATTUNE_CONFIG=/opt/attune/config/config.yaml HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \ CMD curl -f http://localhost:8080/health || exit 1 diff --git a/docker/Dockerfile.sensor.optimized b/docker/Dockerfile.sensor.optimized index 3817dda..3d7d1db 100644 --- a/docker/Dockerfile.sensor.optimized +++ b/docker/Dockerfile.sensor.optimized @@ -96,19 +96,18 @@ RUN apt-get update && apt-get install -y \ && rm -rf /var/lib/apt/lists/* RUN useradd -m -u 1000 attune && \ - mkdir -p /opt/attune/packs /opt/attune/logs /opt/attune/runtime_envs && \ + mkdir -p /opt/attune/packs /opt/attune/logs /opt/attune/runtime_envs /opt/attune/config && \ chown -R attune:attune /opt/attune WORKDIR /opt/attune COPY --from=builder /build/attune-sensor /usr/local/bin/attune-sensor -COPY config.docker.yaml ./config.yaml COPY migrations/ ./migrations/ USER attune ENV RUST_LOG=info -ENV ATTUNE_CONFIG=/opt/attune/config.yaml +ENV ATTUNE_CONFIG=/opt/attune/config/config.yaml HEALTHCHECK --interval=30s --timeout=3s --start-period=20s --retries=3 \ CMD kill -0 1 || exit 1 @@ -159,19 +158,18 @@ RUN pip3 install --no-cache-dir --break-system-packages \ python-dateutil>=2.8.0 RUN useradd -m -u 1000 attune && \ - mkdir -p /opt/attune/packs /opt/attune/logs /opt/attune/runtime_envs && \ + mkdir -p /opt/attune/packs /opt/attune/logs /opt/attune/runtime_envs /opt/attune/config && \ chown -R attune:attune /opt/attune WORKDIR /opt/attune COPY --from=builder /build/attune-sensor /usr/local/bin/attune-sensor -COPY config.docker.yaml ./config.yaml COPY migrations/ ./migrations/ USER attune ENV RUST_LOG=info -ENV ATTUNE_CONFIG=/opt/attune/config.yaml +ENV ATTUNE_CONFIG=/opt/attune/config/config.yaml HEALTHCHECK --interval=30s --timeout=3s --start-period=20s --retries=3 \ CMD kill -0 1 || exit 1 diff --git a/docker/Dockerfile.worker b/docker/Dockerfile.worker deleted file mode 100644 index 7c06f2d..0000000 --- a/docker/Dockerfile.worker +++ /dev/null @@ -1,300 +0,0 @@ -# Multi-stage Dockerfile for Attune workers -# Supports building different worker variants with different runtime capabilities -# -# Usage: -# docker build --target worker-base -t attune-worker:base -f docker/Dockerfile.worker . -# docker build --target worker-python -t attune-worker:python -f docker/Dockerfile.worker . -# docker build --target worker-node -t attune-worker:node -f docker/Dockerfile.worker . -# docker build --target worker-full -t attune-worker:full -f docker/Dockerfile.worker . -# -# BuildKit cache mounts are used to speed up incremental builds. - -ARG RUST_VERSION=1.92 -ARG DEBIAN_VERSION=bookworm -ARG NODE_VERSION=20 - -# ============================================================================ -# Stage 1: Builder - Compile the worker binary -# ============================================================================ -FROM rust:${RUST_VERSION}-${DEBIAN_VERSION} AS builder - -# Install build dependencies -RUN apt-get update && apt-get install -y \ - pkg-config \ - libssl-dev \ - ca-certificates \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /build - -# Increase rustc stack size to prevent SIGSEGV during release builds -ENV RUST_MIN_STACK=16777216 - -# Copy workspace manifests and source code -COPY Cargo.toml Cargo.lock ./ -COPY crates/ ./crates/ -COPY migrations/ ./migrations/ -COPY .sqlx/ ./.sqlx/ - -# Build the worker binary with BuildKit cache mounts -# sharing=locked prevents race conditions during parallel builds -RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ - --mount=type=cache,target=/usr/local/cargo/git,sharing=locked \ - --mount=type=cache,target=/build/target,sharing=locked \ - cargo build --release --bin attune-worker && \ - cp /build/target/release/attune-worker /build/attune-worker - -# Verify the binary was built -RUN ls -lh /build/attune-worker && \ - file /build/attune-worker && \ - /build/attune-worker --version || echo "Version check skipped" - -# ============================================================================ -# Stage 2a: Base Worker (Shell only) -# Runtime capabilities: shell -# Use case: Lightweight workers for shell scripts and basic automation -# ============================================================================ -FROM debian:${DEBIAN_VERSION}-slim AS worker-base - -# Install runtime dependencies -RUN apt-get update && apt-get install -y \ - ca-certificates \ - libssl3 \ - curl \ - bash \ - procps \ - && rm -rf /var/lib/apt/lists/* - -# Create worker user and directories -RUN useradd -m -u 1000 attune && \ - mkdir -p /opt/attune/packs /opt/attune/logs && \ - chown -R attune:attune /opt/attune - -WORKDIR /opt/attune - -# Copy worker binary from builder -COPY --from=builder /build/attune-worker /usr/local/bin/attune-worker - -# Copy configuration template -COPY config.docker.yaml ./config.yaml - -# Copy packs directory -COPY packs/ ./packs/ - -# Set ownership -RUN chown -R attune:attune /opt/attune - -# Switch to non-root user -USER attune - -# Environment variables -ENV ATTUNE_WORKER_RUNTIMES="shell" -ENV ATTUNE_WORKER_TYPE="container" -ENV RUST_LOG=info -ENV ATTUNE_CONFIG=/opt/attune/config.yaml - -# Health check -HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \ - CMD pgrep -f attune-worker || exit 1 - -# Run the worker -CMD ["/usr/local/bin/attune-worker"] - -# ============================================================================ -# Stage 2b: Python Worker (Shell + Python) -# Runtime capabilities: shell, python -# Use case: Python actions and scripts with dependencies -# -# Uses debian-slim + apt python3 (NOT the python: Docker image) so that -# python3 lives at /usr/bin/python3 — the same path as worker-full. -# This avoids broken venv symlinks when multiple workers share the -# runtime_envs volume. -# ============================================================================ -FROM debian:${DEBIAN_VERSION}-slim AS worker-python - -# Install system dependencies including Python -RUN apt-get update && apt-get install -y \ - ca-certificates \ - libssl3 \ - curl \ - build-essential \ - python3 \ - python3-pip \ - python3-venv \ - procps \ - && rm -rf /var/lib/apt/lists/* - -# Create python symlink for convenience -RUN ln -sf /usr/bin/python3 /usr/bin/python - -# Install common Python packages -# Use --break-system-packages for Debian 12+ pip-in-system-python restrictions -RUN pip3 install --no-cache-dir --break-system-packages \ - requests>=2.31.0 \ - pyyaml>=6.0 \ - jinja2>=3.1.0 \ - python-dateutil>=2.8.0 - -# Create worker user and directories -RUN useradd -m -u 1000 attune && \ - mkdir -p /opt/attune/packs /opt/attune/logs /opt/attune/runtime_envs && \ - chown -R attune:attune /opt/attune - -WORKDIR /opt/attune - -# Copy worker binary from builder -COPY --from=builder /build/attune-worker /usr/local/bin/attune-worker - -# Copy configuration template -COPY config.docker.yaml ./config.yaml - -# Copy packs directory -COPY packs/ ./packs/ - -# Set ownership -RUN chown -R attune:attune /opt/attune - -# Switch to non-root user -USER attune - -# Environment variables -ENV ATTUNE_WORKER_RUNTIMES="shell,python" -ENV ATTUNE_WORKER_TYPE="container" -ENV RUST_LOG=info -ENV ATTUNE_CONFIG=/opt/attune/config.yaml - -# Health check -HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \ - CMD pgrep -f attune-worker || exit 1 - -# Run the worker -CMD ["/usr/local/bin/attune-worker"] - -# ============================================================================ -# Stage 2c: Node Worker (Shell + Node.js) -# Runtime capabilities: shell, node -# Use case: JavaScript/TypeScript actions and npm packages -# -# Uses debian-slim + NodeSource apt repo (NOT the node: Docker image) so that -# node lives at /usr/bin/node — the same path as worker-full. -# This avoids path mismatches when multiple workers share volumes. -# ============================================================================ -FROM debian:${DEBIAN_VERSION}-slim AS worker-node - -# Install system dependencies -RUN apt-get update && apt-get install -y \ - ca-certificates \ - libssl3 \ - curl \ - procps \ - && rm -rf /var/lib/apt/lists/* - -# Install Node.js from NodeSource (same method as worker-full) -RUN curl -fsSL https://deb.nodesource.com/setup_${NODE_VERSION}.x | bash - && \ - apt-get install -y nodejs && \ - rm -rf /var/lib/apt/lists/* - -# Create worker user and directories -RUN useradd -m -u 1000 attune && \ - mkdir -p /opt/attune/packs /opt/attune/logs /opt/attune/runtime_envs && \ - chown -R attune:attune /opt/attune - -WORKDIR /opt/attune - -# Copy worker binary from builder -COPY --from=builder /build/attune-worker /usr/local/bin/attune-worker - -# Copy configuration template -COPY config.docker.yaml ./config.yaml - -# Copy packs directory -COPY packs/ ./packs/ - -# Set ownership -RUN chown -R attune:attune /opt/attune - -# Switch to non-root user -USER attune - -# Environment variables -ENV ATTUNE_WORKER_RUNTIMES="shell,node" -ENV ATTUNE_WORKER_TYPE="container" -ENV RUST_LOG=info -ENV ATTUNE_CONFIG=/opt/attune/config.yaml - -# Health check -HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \ - CMD pgrep -f attune-worker || exit 1 - -# Run the worker -CMD ["/usr/local/bin/attune-worker"] - -# ============================================================================ -# Stage 2d: Full Worker (All runtimes) -# Runtime capabilities: shell, python, node, native -# Use case: General-purpose automation with multi-language support -# ============================================================================ -FROM debian:${DEBIAN_VERSION} AS worker-full - -# Install system dependencies including Python and Node.js -RUN apt-get update && apt-get install -y \ - ca-certificates \ - libssl3 \ - curl \ - build-essential \ - python3 \ - python3-pip \ - python3-venv \ - procps \ - && rm -rf /var/lib/apt/lists/* - -# Install Node.js from NodeSource (same method and version as worker-node) -RUN curl -fsSL https://deb.nodesource.com/setup_${NODE_VERSION}.x | bash - && \ - apt-get install -y nodejs && \ - rm -rf /var/lib/apt/lists/* - -# Create python symlink for convenience -RUN ln -sf /usr/bin/python3 /usr/bin/python - -# Install common Python packages -# Use --break-system-packages for Debian 12+ pip-in-system-python restrictions -RUN pip3 install --no-cache-dir --break-system-packages \ - requests>=2.31.0 \ - pyyaml>=6.0 \ - jinja2>=3.1.0 \ - python-dateutil>=2.8.0 - -# Create worker user and directories -RUN useradd -m -u 1000 attune && \ - mkdir -p /opt/attune/packs /opt/attune/logs /opt/attune/runtime_envs && \ - chown -R attune:attune /opt/attune - -WORKDIR /opt/attune - -# Copy worker binary from builder -COPY --from=builder /build/attune-worker /usr/local/bin/attune-worker - -# Copy configuration template -COPY config.docker.yaml ./config.yaml - -# Copy packs directory -COPY packs/ ./packs/ - -# Set ownership -RUN chown -R attune:attune /opt/attune - -# Switch to non-root user -USER attune - -# Environment variables -ENV ATTUNE_WORKER_RUNTIMES="shell,python,node,native" -ENV ATTUNE_WORKER_TYPE="container" -ENV RUST_LOG=info -ENV ATTUNE_CONFIG=/opt/attune/config.yaml - -# Health check -HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \ - CMD pgrep -f attune-worker || exit 1 - -# Run the worker -CMD ["/usr/local/bin/attune-worker"] diff --git a/docker/Dockerfile.worker.optimized b/docker/Dockerfile.worker.optimized index 7bf15e3..367c8ab 100644 --- a/docker/Dockerfile.worker.optimized +++ b/docker/Dockerfile.worker.optimized @@ -101,20 +101,19 @@ RUN apt-get update && apt-get install -y \ && rm -rf /var/lib/apt/lists/* RUN useradd -m -u 1000 attune && \ - mkdir -p /opt/attune/packs /opt/attune/logs /opt/attune/runtime_envs && \ + mkdir -p /opt/attune/packs /opt/attune/logs /opt/attune/runtime_envs /opt/attune/config && \ chown -R attune:attune /opt/attune WORKDIR /opt/attune COPY --from=builder /build/attune-worker /usr/local/bin/attune-worker -COPY config.docker.yaml ./config.yaml USER attune ENV ATTUNE_WORKER_RUNTIMES="shell" ENV ATTUNE_WORKER_TYPE="container" ENV RUST_LOG=info -ENV ATTUNE_CONFIG=/opt/attune/config.yaml +ENV ATTUNE_CONFIG=/opt/attune/config/config.yaml HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \ CMD pgrep -f attune-worker || exit 1 @@ -154,20 +153,19 @@ RUN pip3 install --no-cache-dir --break-system-packages \ python-dateutil>=2.8.0 RUN useradd -m -u 1000 attune && \ - mkdir -p /opt/attune/packs /opt/attune/logs /opt/attune/runtime_envs && \ + mkdir -p /opt/attune/packs /opt/attune/logs /opt/attune/runtime_envs /opt/attune/config && \ chown -R attune:attune /opt/attune WORKDIR /opt/attune COPY --from=builder /build/attune-worker /usr/local/bin/attune-worker -COPY config.docker.yaml ./config.yaml USER attune ENV ATTUNE_WORKER_RUNTIMES="shell,python" ENV ATTUNE_WORKER_TYPE="container" ENV RUST_LOG=info -ENV ATTUNE_CONFIG=/opt/attune/config.yaml +ENV ATTUNE_CONFIG=/opt/attune/config/config.yaml HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \ CMD pgrep -f attune-worker || exit 1 @@ -199,20 +197,19 @@ RUN curl -fsSL https://deb.nodesource.com/setup_${NODE_VERSION}.x | bash - && \ rm -rf /var/lib/apt/lists/* RUN useradd -m -u 1000 attune && \ - mkdir -p /opt/attune/packs /opt/attune/logs /opt/attune/runtime_envs && \ + mkdir -p /opt/attune/packs /opt/attune/logs /opt/attune/runtime_envs /opt/attune/config && \ chown -R attune:attune /opt/attune WORKDIR /opt/attune COPY --from=builder /build/attune-worker /usr/local/bin/attune-worker -COPY config.docker.yaml ./config.yaml USER attune ENV ATTUNE_WORKER_RUNTIMES="shell,node" ENV ATTUNE_WORKER_TYPE="container" ENV RUST_LOG=info -ENV ATTUNE_CONFIG=/opt/attune/config.yaml +ENV ATTUNE_CONFIG=/opt/attune/config/config.yaml HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \ CMD pgrep -f attune-worker || exit 1 @@ -253,20 +250,19 @@ RUN pip3 install --no-cache-dir --break-system-packages \ python-dateutil>=2.8.0 RUN useradd -m -u 1000 attune && \ - mkdir -p /opt/attune/packs /opt/attune/logs /opt/attune/runtime_envs && \ + mkdir -p /opt/attune/packs /opt/attune/logs /opt/attune/runtime_envs /opt/attune/config && \ chown -R attune:attune /opt/attune WORKDIR /opt/attune COPY --from=builder /build/attune-worker /usr/local/bin/attune-worker -COPY config.docker.yaml ./config.yaml USER attune ENV ATTUNE_WORKER_RUNTIMES="shell,python,node,native" ENV ATTUNE_WORKER_TYPE="container" ENV RUST_LOG=info -ENV ATTUNE_CONFIG=/opt/attune/config.yaml +ENV ATTUNE_CONFIG=/opt/attune/config/config.yaml HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \ CMD pgrep -f attune-worker || exit 1 diff --git a/docker/README.md b/docker/README.md index 6ce2311..cbbaeb0 100644 --- a/docker/README.md +++ b/docker/README.md @@ -29,13 +29,16 @@ curl -X POST http://localhost:8080/auth/login \ ### Dockerfiles -- **`Dockerfile`** - Multi-stage Dockerfile for all Rust services (API, Executor, Worker, Sensor, Notifier) +- **`Dockerfile.optimized`** - Multi-stage Dockerfile for Rust services (API, Executor, Notifier) - Uses build argument `SERVICE` to specify which service to build - - Example: `docker build --build-arg SERVICE=api -f docker/Dockerfile -t attune-api .` + - Example: `docker build --build-arg SERVICE=api -f docker/Dockerfile.optimized -t attune-api .` -- **`Dockerfile.worker`** - Multi-stage Dockerfile for containerized workers with different runtime capabilities +- **`Dockerfile.worker.optimized`** - Multi-stage Dockerfile for containerized workers with different runtime capabilities - Supports 4 variants: `worker-base`, `worker-python`, `worker-node`, `worker-full` - See [README.worker.md](./README.worker.md) for details + +- **`Dockerfile.sensor.optimized`** - Multi-stage Dockerfile for the sensor service + - Supports `sensor-base` and `sensor-full` - **`Dockerfile.web`** - Multi-stage Dockerfile for React Web UI - Builds with Node.js and serves with Nginx @@ -130,7 +133,7 @@ docker compose build worker DOCKER_BUILDKIT=1 docker build \ --build-arg SERVICE=api \ --build-arg RUST_VERSION=1.92 \ - -f docker/Dockerfile \ + -f docker/Dockerfile.optimized \ -t attune-api:custom \ . ``` @@ -403,9 +406,9 @@ Caused by: lock file version `4` was found, but this version of Cargo does not understand this lock file ``` -Solution: Update Rust version in Dockerfile +Solution: Update Rust version in the optimized Dockerfile ```bash -# Edit docker/Dockerfile and change: +# Edit docker/Dockerfile.optimized and change: ARG RUST_VERSION=1.75 # to: ARG RUST_VERSION=1.92 @@ -557,4 +560,4 @@ docker system prune -a --volumes - [Docker Compose Documentation](https://docs.docker.com/compose/) - [Multi-stage Builds](https://docs.docker.com/build/building/multi-stage/) - [Dockerfile Best Practices](https://docs.docker.com/develop/dev-best-practices/) -- [Main Documentation](../docs/docker-deployment.md) \ No newline at end of file +- [Main Documentation](../docs/docker-deployment.md) diff --git a/docker/README.worker.md b/docker/README.worker.md index 208c048..21412f7 100644 --- a/docker/README.worker.md +++ b/docker/README.worker.md @@ -67,7 +67,7 @@ make docker-build-worker-full DOCKER_BUILDKIT=1 docker build \ --target worker-python \ -t attune-worker:python \ - -f docker/Dockerfile.worker \ + -f docker/Dockerfile.worker.optimized \ . ``` @@ -202,7 +202,7 @@ ENV ATTUNE_WORKER_RUNTIMES="shell,ruby" ### Multi-stage Build -The `Dockerfile.worker` uses a multi-stage build pattern: +The `Dockerfile.worker.optimized` uses a multi-stage build pattern: 1. **Builder Stage**: Compiles the Rust worker binary - Uses BuildKit cache mounts for fast incremental builds @@ -326,7 +326,7 @@ WHERE status = 'active'; ## Files -- `Dockerfile.worker` - Multi-stage worker Dockerfile with all variants +- `Dockerfile.worker.optimized` - Multi-stage worker Dockerfile with all variants - `README.worker.md` - This file - `../docker-compose.yaml` - Service definitions for all workers diff --git a/docker/enable-buildkit.sh b/docker/enable-buildkit.sh index bb61d48..e12da5c 100755 --- a/docker/enable-buildkit.sh +++ b/docker/enable-buildkit.sh @@ -171,7 +171,7 @@ echo " export DOCKER_BUILDKIT=1" echo " docker compose build" echo "" echo "2. Build individual service:" -echo " DOCKER_BUILDKIT=1 docker build --build-arg SERVICE=api -f docker/Dockerfile -t attune-api ." +echo " DOCKER_BUILDKIT=1 docker build --build-arg SERVICE=api -f docker/Dockerfile.optimized -t attune-api ." echo "" echo "3. Use Makefile:" echo " export DOCKER_BUILDKIT=1"