diff --git a/AGENTS.md b/AGENTS.md index 249c7a4..da33903 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -99,7 +99,7 @@ docker compose -f docker-compose.yaml -f docker-compose.agent.yaml up -d # Star **Key environment overrides**: `JWT_SECRET`, `ENCRYPTION_KEY` (required for production) ### Docker Build Optimization -- **Optimized Dockerfiles**: `docker/Dockerfile.optimized`, `docker/Dockerfile.worker.optimized`, `docker/Dockerfile.sensor.optimized`, and `docker/Dockerfile.agent` +- **Active Dockerfiles**: `docker/Dockerfile.optimized`, `docker/Dockerfile.agent`, `docker/Dockerfile.web`, and `docker/Dockerfile.pack-binaries` - **Agent Dockerfile** (`docker/Dockerfile.agent`): Builds a statically-linked `attune-agent` binary using musl (`x86_64-unknown-linux-musl`). Three stages: `builder` (cross-compile), `agent-binary` (scratch — just the binary), `agent-init` (busybox — for volume population via `cp`). The binary has zero runtime dependencies (no glibc, no libssl). Build with `make docker-build-agent`. - **Strategy**: Selective crate copying - only copy crates needed for each service (not entire workspace) - **Performance**: 90% faster incremental builds (~30 sec vs ~5 min for code changes) @@ -681,9 +681,8 @@ When reporting, ask: "Should I fix this first or continue with [original task]?" - `Cargo.toml` - Workspace dependencies - `Makefile` - Development commands - `docker/Dockerfile.optimized` - Optimized service builds (api, executor, notifier) -- `docker/Dockerfile.worker.optimized` - Optimized worker builds (shell, python, node, full) -- `docker/Dockerfile.sensor.optimized` - Optimized sensor builds (base, full) - `docker/Dockerfile.agent` - Statically-linked agent binary (musl, for injection into any container) +- `docker/Dockerfile.web` - Web UI build - `docker/Dockerfile.pack-binaries` - Separate pack binary builder - `scripts/build-pack-binaries.sh` - Build pack binaries script diff --git a/Makefile b/Makefile index 5d2b5ee..acdcf37 100644 --- a/Makefile +++ b/Makefile @@ -237,30 +237,6 @@ docker-build-api: docker-build-web: docker compose build web -# Build worker images -docker-build-workers: docker-build-worker-base docker-build-worker-python docker-build-worker-node docker-build-worker-full - @echo "✅ All worker images built successfully" - -docker-build-worker-base: - @echo "Building base worker (shell only)..." - DOCKER_BUILDKIT=1 docker build --target worker-base -t attune-worker:base -f docker/Dockerfile.worker.optimized . - @echo "✅ Base worker image built: attune-worker:base" - -docker-build-worker-python: - @echo "Building Python worker (shell + python)..." - DOCKER_BUILDKIT=1 docker build --target worker-python -t attune-worker:python -f docker/Dockerfile.worker.optimized . - @echo "✅ Python worker image built: attune-worker:python" - -docker-build-worker-node: - @echo "Building Node.js worker (shell + node)..." - DOCKER_BUILDKIT=1 docker build --target worker-node -t attune-worker:node -f docker/Dockerfile.worker.optimized . - @echo "✅ Node.js worker image built: attune-worker:node" - -docker-build-worker-full: - @echo "Building full worker (all runtimes)..." - DOCKER_BUILDKIT=1 docker build --target worker-full -t attune-worker:full -f docker/Dockerfile.worker.optimized . - @echo "✅ Full worker image built: attune-worker:full" - # Agent binary (statically-linked for injection into any container) build-agent: @echo "Installing musl target (if not already installed)..." diff --git a/docker/Dockerfile.agent b/docker/Dockerfile.agent index a097fa9..f1d1c2d 100644 --- a/docker/Dockerfile.agent +++ b/docker/Dockerfile.agent @@ -56,7 +56,8 @@ ENV SQLX_OFFLINE=true # --------------------------------------------------------------------------- # Dependency caching layer # Copy only Cargo metadata first so `cargo fetch` is cached when only source -# code changes. This follows the same pattern as Dockerfile.worker.optimized. +# code changes. This follows the same selective-copy optimization pattern as +# the other active Dockerfiles in this directory. # --------------------------------------------------------------------------- COPY Cargo.toml Cargo.lock ./ COPY crates/common/Cargo.toml ./crates/common/Cargo.toml diff --git a/docker/Dockerfile.init-packs b/docker/Dockerfile.init-packs deleted file mode 100644 index 3732bfc..0000000 --- a/docker/Dockerfile.init-packs +++ /dev/null @@ -1,10 +0,0 @@ -FROM python:3.11-slim - -COPY packs /source/packs -COPY scripts/load_core_pack.py /scripts/load_core_pack.py -COPY docker/init-packs.sh /init-packs.sh - -RUN pip install --no-cache-dir psycopg2-binary pyyaml && \ - chmod +x /init-packs.sh - -CMD ["/bin/sh", "/init-packs.sh"] diff --git a/docker/Dockerfile.init-user b/docker/Dockerfile.init-user deleted file mode 100644 index 110139d..0000000 --- a/docker/Dockerfile.init-user +++ /dev/null @@ -1,7 +0,0 @@ -FROM postgres:16-alpine - -COPY docker/init-user.sh /init-user.sh - -RUN chmod +x /init-user.sh - -CMD ["/bin/sh", "/init-user.sh"] diff --git a/docker/Dockerfile.migrations b/docker/Dockerfile.migrations deleted file mode 100644 index 7442fb1..0000000 --- a/docker/Dockerfile.migrations +++ /dev/null @@ -1,9 +0,0 @@ -FROM postgres:16-alpine - -COPY migrations /migrations -COPY docker/run-migrations.sh /run-migrations.sh -COPY docker/init-roles.sql /docker/init-roles.sql - -RUN chmod +x /run-migrations.sh - -CMD ["/bin/sh", "/run-migrations.sh"] diff --git a/docker/Dockerfile.sensor.optimized b/docker/Dockerfile.sensor.optimized deleted file mode 100644 index 66a122e..0000000 --- a/docker/Dockerfile.sensor.optimized +++ /dev/null @@ -1,177 +0,0 @@ -# Multi-stage Dockerfile for Attune sensor service -# -# Simple and robust: build the entire workspace, then copy the sensor binary -# into different runtime base images depending on language support needed. -# -# Targets: -# sensor-base - Native sensors only (lightweight) -# sensor-full - Native + Python + Node.js sensors -# -# Usage: -# DOCKER_BUILDKIT=1 docker build --target sensor-base -t attune-sensor:base -f docker/Dockerfile.sensor.optimized . -# DOCKER_BUILDKIT=1 docker build --target sensor-full -t attune-sensor:full -f docker/Dockerfile.sensor.optimized . -# -# Note: Packs are NOT copied into the image — they are mounted as volumes at runtime. - -ARG RUST_VERSION=1.92 -ARG DEBIAN_VERSION=bookworm -ARG NODE_VERSION=20 - -# ============================================================================ -# Stage 1: Builder - Compile the entire workspace -# ============================================================================ -FROM rust:${RUST_VERSION}-${DEBIAN_VERSION} AS builder - -RUN apt-get update && apt-get install -y \ - pkg-config \ - libssl-dev \ - ca-certificates \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /build - -# Increase rustc stack size to prevent SIGSEGV during release builds -ENV RUST_MIN_STACK=67108864 - -# Copy dependency metadata first so `cargo fetch` layer is cached -# when only source code changes (Cargo.toml/Cargo.lock stay the same) -COPY Cargo.toml Cargo.lock ./ -COPY crates/common/Cargo.toml ./crates/common/Cargo.toml -COPY crates/api/Cargo.toml ./crates/api/Cargo.toml -COPY crates/executor/Cargo.toml ./crates/executor/Cargo.toml -COPY crates/sensor/Cargo.toml ./crates/sensor/Cargo.toml -COPY crates/core-timer-sensor/Cargo.toml ./crates/core-timer-sensor/Cargo.toml -COPY crates/worker/Cargo.toml ./crates/worker/Cargo.toml -COPY crates/notifier/Cargo.toml ./crates/notifier/Cargo.toml -COPY crates/cli/Cargo.toml ./crates/cli/Cargo.toml - -# Create minimal stub sources so cargo can resolve the workspace and fetch deps. -# These are ONLY used for `cargo fetch` — never compiled. -RUN mkdir -p crates/common/src && echo "" > crates/common/src/lib.rs && \ - mkdir -p crates/api/src && echo "fn main(){}" > crates/api/src/main.rs && \ - mkdir -p crates/executor/src && echo "fn main(){}" > crates/executor/src/main.rs && \ - mkdir -p crates/executor/benches && echo "fn main(){}" > crates/executor/benches/context_clone.rs && \ - mkdir -p crates/sensor/src && echo "fn main(){}" > crates/sensor/src/main.rs && \ - mkdir -p crates/core-timer-sensor/src && echo "fn main(){}" > crates/core-timer-sensor/src/main.rs && \ - mkdir -p crates/worker/src && echo "fn main(){}" > crates/worker/src/main.rs && \ - echo "fn main(){}" > crates/worker/src/agent_main.rs && \ - mkdir -p crates/notifier/src && echo "fn main(){}" > crates/notifier/src/main.rs && \ - mkdir -p crates/cli/src && echo "fn main(){}" > crates/cli/src/main.rs - -# Download all dependencies (cached unless Cargo.toml/Cargo.lock change) -RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=shared \ - --mount=type=cache,target=/usr/local/cargo/git,sharing=shared \ - cargo fetch - -# Now copy the real source code and migrations -COPY migrations/ ./migrations/ -COPY crates/ ./crates/ - -# Build the entire workspace in release mode. -# All binaries are compiled together, sharing dependency compilation. -# target cache uses sharing=locked so concurrent service builds serialize -# writes to the shared compilation cache instead of corrupting it. -RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=shared \ - --mount=type=cache,target=/usr/local/cargo/git,sharing=shared \ - --mount=type=cache,target=/build/target,sharing=locked \ - cargo build --release --workspace --bins -j 4 && \ - cp /build/target/release/attune-sensor /build/attune-sensor - -# Verify the binary was built -RUN ls -lh /build/attune-sensor && \ - file /build/attune-sensor - -# ============================================================================ -# Stage 2a: Base Sensor (Native sensors only) -# Runtime capabilities: native binary sensors -# ============================================================================ -FROM debian:${DEBIAN_VERSION}-slim AS sensor-base - -RUN apt-get update && apt-get install -y \ - ca-certificates \ - libssl3 \ - curl \ - bash \ - procps \ - && rm -rf /var/lib/apt/lists/* - -RUN useradd -m -u 1000 attune && \ - mkdir -p /opt/attune/packs /opt/attune/logs /opt/attune/runtime_envs /opt/attune/config && \ - chown -R attune:attune /opt/attune - -WORKDIR /opt/attune - -COPY --from=builder /build/attune-sensor /usr/local/bin/attune-sensor -COPY migrations/ ./migrations/ - -USER attune - -ENV RUST_LOG=info -ENV ATTUNE_CONFIG=/opt/attune/config/config.yaml - -HEALTHCHECK --interval=30s --timeout=3s --start-period=20s --retries=3 \ - CMD kill -0 1 || exit 1 - -CMD ["/usr/local/bin/attune-sensor"] - -# ============================================================================ -# Stage 2b: Full Sensor (Native + Python + Node.js sensors) -# Runtime capabilities: native, python, node -# -# Uses debian-slim + apt python3 + NodeSource node so that interpreter -# paths (/usr/bin/python3, /usr/bin/node) are identical to the worker -# containers. This avoids broken symlinks and path mismatches when -# sensors and workers share the runtime_envs volume. -# ============================================================================ -FROM debian:${DEBIAN_VERSION}-slim AS sensor-full - -# Re-declare global ARG so it's available in RUN commands within this stage -# (global ARGs are only automatically available in FROM instructions) -ARG NODE_VERSION=20 - -RUN apt-get update && apt-get install -y \ - ca-certificates \ - libssl3 \ - curl \ - bash \ - build-essential \ - python3 \ - python3-pip \ - python3-venv \ - procps \ - && rm -rf /var/lib/apt/lists/* - -# Install Node.js from NodeSource (same method and version as workers) -RUN curl -fsSL https://deb.nodesource.com/setup_${NODE_VERSION}.x | bash - && \ - apt-get install -y nodejs && \ - rm -rf /var/lib/apt/lists/* - -# Create python symlink for convenience -RUN ln -sf /usr/bin/python3 /usr/bin/python - -# Install common Python packages used by sensor scripts -# Use --break-system-packages for Debian 12+ pip-in-system-python restrictions -RUN pip3 install --no-cache-dir --break-system-packages \ - requests>=2.31.0 \ - pyyaml>=6.0 \ - jinja2>=3.1.0 \ - python-dateutil>=2.8.0 - -RUN useradd -m -u 1000 attune && \ - mkdir -p /opt/attune/packs /opt/attune/logs /opt/attune/runtime_envs /opt/attune/config && \ - chown -R attune:attune /opt/attune - -WORKDIR /opt/attune - -COPY --from=builder /build/attune-sensor /usr/local/bin/attune-sensor -COPY migrations/ ./migrations/ - -USER attune - -ENV RUST_LOG=info -ENV ATTUNE_CONFIG=/opt/attune/config/config.yaml - -HEALTHCHECK --interval=30s --timeout=3s --start-period=20s --retries=3 \ - CMD kill -0 1 || exit 1 - -CMD ["/usr/local/bin/attune-sensor"] diff --git a/docker/Dockerfile.worker.optimized b/docker/Dockerfile.worker.optimized deleted file mode 100644 index b64ee38..0000000 --- a/docker/Dockerfile.worker.optimized +++ /dev/null @@ -1,270 +0,0 @@ -# Multi-stage Dockerfile for Attune worker service -# -# Simple and robust: build the entire workspace, then copy the worker binary -# into different runtime base images depending on language support needed. -# No dummy source compilation, no selective crate copying, no fragile hacks. -# -# Targets: -# worker-base - Shell only (lightweight) -# worker-python - Shell + Python -# worker-node - Shell + Node.js -# worker-full - Shell + Python + Node.js + Native -# -# Usage: -# DOCKER_BUILDKIT=1 docker build --target worker-base -t attune-worker:base -f docker/Dockerfile.worker.optimized . -# DOCKER_BUILDKIT=1 docker build --target worker-python -t attune-worker:python -f docker/Dockerfile.worker.optimized . -# DOCKER_BUILDKIT=1 docker build --target worker-node -t attune-worker:node -f docker/Dockerfile.worker.optimized . -# DOCKER_BUILDKIT=1 docker build --target worker-full -t attune-worker:full -f docker/Dockerfile.worker.optimized . -# -# Note: Packs are NOT copied into the image — they are mounted as volumes at runtime. - -ARG RUST_VERSION=1.92 -ARG DEBIAN_VERSION=bookworm -ARG NODE_VERSION=20 - -# ============================================================================ -# Stage 1: Builder - Compile the entire workspace -# ============================================================================ -FROM rust:${RUST_VERSION}-${DEBIAN_VERSION} AS builder - -RUN apt-get update && apt-get install -y \ - pkg-config \ - libssl-dev \ - ca-certificates \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /build - -# Increase rustc stack size to prevent SIGSEGV during release builds -ENV RUST_MIN_STACK=67108864 - -# Copy dependency metadata first so `cargo fetch` layer is cached -# when only source code changes (Cargo.toml/Cargo.lock stay the same) -COPY Cargo.toml Cargo.lock ./ -COPY crates/common/Cargo.toml ./crates/common/Cargo.toml -COPY crates/api/Cargo.toml ./crates/api/Cargo.toml -COPY crates/executor/Cargo.toml ./crates/executor/Cargo.toml -COPY crates/sensor/Cargo.toml ./crates/sensor/Cargo.toml -COPY crates/core-timer-sensor/Cargo.toml ./crates/core-timer-sensor/Cargo.toml -COPY crates/worker/Cargo.toml ./crates/worker/Cargo.toml -COPY crates/notifier/Cargo.toml ./crates/notifier/Cargo.toml -COPY crates/cli/Cargo.toml ./crates/cli/Cargo.toml - -# Create minimal stub sources so cargo can resolve the workspace and fetch deps. -# Unlike the old approach, these are ONLY used for `cargo fetch` — never compiled. -RUN mkdir -p crates/common/src && echo "" > crates/common/src/lib.rs && \ - mkdir -p crates/api/src && echo "fn main(){}" > crates/api/src/main.rs && \ - mkdir -p crates/executor/src && echo "fn main(){}" > crates/executor/src/main.rs && \ - mkdir -p crates/executor/benches && echo "fn main(){}" > crates/executor/benches/context_clone.rs && \ - mkdir -p crates/sensor/src && echo "fn main(){}" > crates/sensor/src/main.rs && \ - mkdir -p crates/core-timer-sensor/src && echo "fn main(){}" > crates/core-timer-sensor/src/main.rs && \ - mkdir -p crates/worker/src && echo "fn main(){}" > crates/worker/src/main.rs && \ - echo "fn main(){}" > crates/worker/src/agent_main.rs && \ - mkdir -p crates/notifier/src && echo "fn main(){}" > crates/notifier/src/main.rs && \ - mkdir -p crates/cli/src && echo "fn main(){}" > crates/cli/src/main.rs - -# Download all dependencies (cached unless Cargo.toml/Cargo.lock change) -RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=shared \ - --mount=type=cache,target=/usr/local/cargo/git,sharing=shared \ - cargo fetch - -# Now copy the real source code and migrations -COPY migrations/ ./migrations/ -COPY crates/ ./crates/ - -# Build the entire workspace in release mode. -# All binaries are compiled together, sharing dependency compilation. -# target cache uses sharing=locked so concurrent service builds serialize -# writes to the shared compilation cache instead of corrupting it. -RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=shared \ - --mount=type=cache,target=/usr/local/cargo/git,sharing=shared \ - --mount=type=cache,target=/build/target,sharing=locked \ - cargo build --release --workspace --bins -j 4 && \ - cp /build/target/release/attune-worker /build/attune-worker - -# Verify the binary was built -RUN ls -lh /build/attune-worker && \ - file /build/attune-worker - -# ============================================================================ -# Stage 2a: Base Worker (Shell only) -# Runtime capabilities: shell -# ============================================================================ -FROM debian:${DEBIAN_VERSION}-slim AS worker-base - -RUN apt-get update && apt-get install -y \ - ca-certificates \ - libssl3 \ - curl \ - bash \ - procps \ - && rm -rf /var/lib/apt/lists/* - -RUN useradd -m -u 1000 attune && \ - mkdir -p /opt/attune/packs /opt/attune/logs /opt/attune/runtime_envs /opt/attune/config && \ - chown -R attune:attune /opt/attune - -WORKDIR /opt/attune - -COPY --from=builder /build/attune-worker /usr/local/bin/attune-worker - -USER attune - -ENV ATTUNE_WORKER_RUNTIMES="shell" -ENV ATTUNE_WORKER_TYPE="container" -ENV RUST_LOG=info -ENV ATTUNE_CONFIG=/opt/attune/config/config.yaml - -HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \ - CMD pgrep -f attune-worker || exit 1 - -CMD ["/usr/local/bin/attune-worker"] - -# ============================================================================ -# Stage 2b: Python Worker (Shell + Python) -# Runtime capabilities: shell, python -# -# Uses debian-slim + apt python3 (NOT the python: Docker image) so that -# python3 lives at /usr/bin/python3 — the same path as worker-full. -# This avoids broken venv symlinks when multiple workers share the -# runtime_envs volume. -# ============================================================================ -FROM debian:${DEBIAN_VERSION}-slim AS worker-python - -RUN apt-get update && apt-get install -y \ - ca-certificates \ - libssl3 \ - curl \ - build-essential \ - python3 \ - python3-pip \ - python3-venv \ - procps \ - && rm -rf /var/lib/apt/lists/* - -# Create python symlink for convenience -RUN ln -sf /usr/bin/python3 /usr/bin/python - -# Use --break-system-packages for Debian 12+ pip-in-system-python restrictions -RUN pip3 install --no-cache-dir --break-system-packages \ - requests>=2.31.0 \ - pyyaml>=6.0 \ - jinja2>=3.1.0 \ - python-dateutil>=2.8.0 - -RUN useradd -m -u 1000 attune && \ - mkdir -p /opt/attune/packs /opt/attune/logs /opt/attune/runtime_envs /opt/attune/config && \ - chown -R attune:attune /opt/attune - -WORKDIR /opt/attune - -COPY --from=builder /build/attune-worker /usr/local/bin/attune-worker - -USER attune - -ENV ATTUNE_WORKER_RUNTIMES="shell,python" -ENV ATTUNE_WORKER_TYPE="container" -ENV RUST_LOG=info -ENV ATTUNE_CONFIG=/opt/attune/config/config.yaml - -HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \ - CMD pgrep -f attune-worker || exit 1 - -CMD ["/usr/local/bin/attune-worker"] - -# ============================================================================ -# Stage 2c: Node Worker (Shell + Node.js) -# Runtime capabilities: shell, node -# -# Uses debian-slim + NodeSource apt repo (NOT the node: Docker image) so that -# node lives at /usr/bin/node — the same path as worker-full. -# This avoids path mismatches when multiple workers share volumes. -# ============================================================================ -FROM debian:${DEBIAN_VERSION}-slim AS worker-node - -ARG NODE_VERSION=20 - -RUN apt-get update && apt-get install -y \ - ca-certificates \ - libssl3 \ - curl \ - procps \ - && rm -rf /var/lib/apt/lists/* - -# Install Node.js from NodeSource (same method as worker-full) -RUN curl -fsSL https://deb.nodesource.com/setup_${NODE_VERSION}.x | bash - && \ - apt-get install -y nodejs && \ - rm -rf /var/lib/apt/lists/* - -RUN useradd -m -u 1000 attune && \ - mkdir -p /opt/attune/packs /opt/attune/logs /opt/attune/runtime_envs /opt/attune/config && \ - chown -R attune:attune /opt/attune - -WORKDIR /opt/attune - -COPY --from=builder /build/attune-worker /usr/local/bin/attune-worker - -USER attune - -ENV ATTUNE_WORKER_RUNTIMES="shell,node" -ENV ATTUNE_WORKER_TYPE="container" -ENV RUST_LOG=info -ENV ATTUNE_CONFIG=/opt/attune/config/config.yaml - -HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \ - CMD pgrep -f attune-worker || exit 1 - -CMD ["/usr/local/bin/attune-worker"] - -# ============================================================================ -# Stage 2d: Full Worker (All runtimes) -# Runtime capabilities: shell, python, node, native -# ============================================================================ -FROM debian:${DEBIAN_VERSION} AS worker-full - -ARG NODE_VERSION=20 - -RUN apt-get update && apt-get install -y \ - ca-certificates \ - libssl3 \ - curl \ - build-essential \ - python3 \ - python3-pip \ - python3-venv \ - procps \ - && rm -rf /var/lib/apt/lists/* - -# Install Node.js from NodeSource (same method and version as worker-node) -RUN curl -fsSL https://deb.nodesource.com/setup_${NODE_VERSION}.x | bash - && \ - apt-get install -y nodejs && \ - rm -rf /var/lib/apt/lists/* - -RUN ln -sf /usr/bin/python3 /usr/bin/python - -# Use --break-system-packages for Debian 12+ pip-in-system-python restrictions -RUN pip3 install --no-cache-dir --break-system-packages \ - requests>=2.31.0 \ - pyyaml>=6.0 \ - jinja2>=3.1.0 \ - python-dateutil>=2.8.0 - -RUN useradd -m -u 1000 attune && \ - mkdir -p /opt/attune/packs /opt/attune/logs /opt/attune/runtime_envs /opt/attune/config && \ - chown -R attune:attune /opt/attune - -WORKDIR /opt/attune - -COPY --from=builder /build/attune-worker /usr/local/bin/attune-worker - -USER attune - -ENV ATTUNE_WORKER_RUNTIMES="shell,python,node,native" -ENV ATTUNE_WORKER_TYPE="container" -ENV RUST_LOG=info -ENV ATTUNE_CONFIG=/opt/attune/config/config.yaml - -HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \ - CMD pgrep -f attune-worker || exit 1 - -CMD ["/usr/local/bin/attune-worker"] diff --git a/docker/README.md b/docker/README.md index 52a02c4..79a52c8 100644 --- a/docker/README.md +++ b/docker/README.md @@ -33,13 +33,11 @@ curl -X POST http://localhost:8080/auth/login \ - Uses build argument `SERVICE` to specify which service to build - Example: `docker build --build-arg SERVICE=api -f docker/Dockerfile.optimized -t attune-api .` -- **`Dockerfile.worker.optimized`** - Multi-stage Dockerfile for containerized workers with different runtime capabilities - - Supports 4 variants: `worker-base`, `worker-python`, `worker-node`, `worker-full` - - See [README.worker.md](./README.worker.md) for details +- **`Dockerfile.agent`** - Multi-stage Dockerfile for the statically-linked agent image + - Builds the `agent-init` image used to populate the shared agent binary volume + +- **`Dockerfile.pack-binaries`** - Pack binary builder used by `scripts/build-pack-binaries.sh` -- **`Dockerfile.sensor.optimized`** - Multi-stage Dockerfile for the sensor service - - Supports `sensor-base` and `sensor-full` - - **`Dockerfile.web`** - Multi-stage Dockerfile for React Web UI - Builds with Node.js and serves with Nginx - Includes runtime environment variable injection @@ -122,8 +120,8 @@ docker compose build api # Web UI docker compose build web -# Worker service -docker compose build worker +# Notifier service +docker compose build notifier ``` ### Build with Custom Args diff --git a/docker/README.worker.md b/docker/README.worker.md deleted file mode 100644 index 21412f7..0000000 --- a/docker/README.worker.md +++ /dev/null @@ -1,364 +0,0 @@ -# Attune Worker Containers - -This directory contains Docker configurations for building Attune worker containers with different runtime capabilities. - -## Overview - -Attune workers can run in containers with specialized runtime environments. Workers automatically declare their capabilities when they register with the system, enabling intelligent action scheduling based on runtime requirements. - -## Worker Variants - -### Base Worker (`worker-base`) -- **Runtimes**: `shell` -- **Base Image**: Debian Bookworm Slim -- **Size**: ~580 MB -- **Use Case**: Lightweight workers for shell scripts and basic automation -- **Build**: `make docker-build-worker-base` - -### Python Worker (`worker-python`) -- **Runtimes**: `shell`, `python` -- **Base Image**: Python 3.11 Slim -- **Size**: ~1.2 GB -- **Includes**: pip, virtualenv, common Python libraries (requests, pyyaml, jinja2, python-dateutil) -- **Use Case**: Python actions and scripts with dependencies -- **Build**: `make docker-build-worker-python` - -### Node.js Worker (`worker-node`) -- **Runtimes**: `shell`, `node` -- **Base Image**: Node 20 Slim -- **Size**: ~760 MB -- **Includes**: npm, yarn -- **Use Case**: JavaScript/TypeScript actions and npm packages -- **Build**: `make docker-build-worker-node` - -### Full Worker (`worker-full`) -- **Runtimes**: `shell`, `python`, `node`, `native` -- **Base Image**: Debian Bookworm -- **Size**: ~1.6 GB -- **Includes**: Python 3.x, Node.js 20, build tools -- **Use Case**: General-purpose automation requiring multiple runtimes -- **Build**: `make docker-build-worker-full` - -## Building Worker Images - -### Build All Variants -```bash -make docker-build-workers -``` - -### Build Individual Variants -```bash -# Base worker (shell only) -make docker-build-worker-base - -# Python worker -make docker-build-worker-python - -# Node.js worker -make docker-build-worker-node - -# Full worker (all runtimes) -make docker-build-worker-full -``` - -### Direct Docker Build -```bash -# Using Docker directly with BuildKit -DOCKER_BUILDKIT=1 docker build \ - --target worker-python \ - -t attune-worker:python \ - -f docker/Dockerfile.worker.optimized \ - . -``` - -## Running Workers - -### Using Docker Compose -```bash -# Start specific worker type -docker-compose up -d worker-python - -# Start all workers -docker-compose up -d worker-shell worker-python worker-node worker-full - -# Scale workers -docker-compose up -d --scale worker-python=3 -``` - -### Using Docker Run -```bash -docker run -d \ - --name worker-python-01 \ - --network attune_attune-network \ - -e ATTUNE_WORKER_NAME=worker-python-01 \ - -e ATTUNE_WORKER_RUNTIMES=shell,python \ - -e ATTUNE__DATABASE__URL=postgresql://attune:attune@postgres:5432/attune \ - -e ATTUNE__MESSAGE_QUEUE__URL=amqp://attune:attune@rabbitmq:5672 \ - -v $(pwd)/packs:/opt/attune/packs:ro \ - attune-worker:python -``` - -## Runtime Capability Declaration - -Workers declare their capabilities in three ways (in order of precedence): - -### 1. Environment Variable (Highest Priority) -```bash -ATTUNE_WORKER_RUNTIMES="shell,python,custom" -``` - -### 2. Configuration File -```yaml -worker: - capabilities: - runtimes: ["shell", "python"] -``` - -### 3. Auto-Detection (Fallback) -Workers automatically detect available runtimes by checking for binaries: -- `python3` or `python` → adds `python` -- `node` → adds `node` -- Always includes `shell` and `native` - -## Configuration - -### Key Environment Variables - -| Variable | Description | Example | -|----------|-------------|---------| -| `ATTUNE_WORKER_NAME` | Unique worker identifier | `worker-python-01` | -| `ATTUNE_WORKER_RUNTIMES` | Comma-separated runtime list | `shell,python` | -| `ATTUNE_WORKER_TYPE` | Worker type | `container` | -| `ATTUNE__DATABASE__URL` | PostgreSQL connection | `postgresql://...` | -| `ATTUNE__MESSAGE_QUEUE__URL` | RabbitMQ connection | `amqp://...` | -| `RUST_LOG` | Log level | `info`, `debug`, `trace` | - -### Resource Limits - -Set CPU and memory limits in `docker-compose.override.yml`: - -```yaml -services: - worker-python: - deploy: - resources: - limits: - cpus: '2.0' - memory: 2G - reservations: - cpus: '0.5' - memory: 512M -``` - -## Custom Worker Images - -### Extend Python Worker - -Create a custom worker with additional packages: - -```dockerfile -# Dockerfile.worker.ml -FROM attune-worker:python - -USER root - -# Install ML packages -RUN pip install --no-cache-dir \ - pandas \ - numpy \ - scikit-learn \ - torch - -USER attune - -ENV ATTUNE_WORKER_RUNTIMES="shell,python,ml" -``` - -Build and run: -```bash -docker build -t attune-worker:ml -f Dockerfile.worker.ml . -docker run -d --name worker-ml-01 ... attune-worker:ml -``` - -### Add New Runtime - -Example: Adding Ruby support - -```dockerfile -FROM attune-worker:base - -USER root - -RUN apt-get update && apt-get install -y \ - ruby-full \ - && rm -rf /var/lib/apt/lists/* - -USER attune - -ENV ATTUNE_WORKER_RUNTIMES="shell,ruby" -``` - -## Architecture - -### Multi-stage Build - -The `Dockerfile.worker.optimized` uses a multi-stage build pattern: - -1. **Builder Stage**: Compiles the Rust worker binary - - Uses BuildKit cache mounts for fast incremental builds - - Shared across all worker variants - -2. **Runtime Stages**: Creates specialized worker images - - `worker-base`: Minimal shell runtime - - `worker-python`: Python runtime - - `worker-node`: Node.js runtime - - `worker-full`: All runtimes - -### Build Cache - -BuildKit cache mounts dramatically speed up builds: -- First build: ~5-6 minutes -- Incremental builds: ~30-60 seconds - -Cache is shared across builds using `sharing=locked` to prevent race conditions. - -## Security - -### Non-root Execution -All workers run as user `attune` (UID 1000) - -### Read-only Packs -Pack files are mounted read-only to prevent modification: -```yaml -volumes: - - ./packs:/opt/attune/packs:ro # :ro = read-only -``` - -### Network Isolation -Workers run in isolated Docker network with only necessary service access - -### Secret Management -Use environment variables for sensitive data; never hardcode in images - -## Monitoring - -### Check Worker Registration -```bash -docker-compose exec postgres psql -U attune -d attune -c \ - "SELECT name, worker_type, status, capabilities->>'runtimes' as runtimes FROM worker;" -``` - -### View Logs -```bash -docker-compose logs -f worker-python -``` - -### Check Resource Usage -```bash -docker stats attune-worker-python -``` - -### Verify Health -```bash -docker-compose ps | grep worker -``` - -## Troubleshooting - -### Worker Not Registering - -**Check database connectivity:** -```bash -docker-compose logs worker-python | grep -i database -``` - -**Verify environment:** -```bash -docker-compose exec worker-python env | grep ATTUNE -``` - -### Runtime Not Detected - -**Check runtime availability:** -```bash -docker-compose exec worker-python python3 --version -docker-compose exec worker-python node --version -``` - -**Force runtime declaration:** -```bash -ATTUNE_WORKER_RUNTIMES=shell,python -``` - -### Actions Not Scheduled - -**Verify runtime match:** -```sql --- Check action runtime requirement -SELECT a.ref, r.name as runtime -FROM action a -JOIN runtime r ON a.runtime = r.id -WHERE a.ref = 'core.my_action'; - --- Check worker capabilities -SELECT name, capabilities->>'runtimes' -FROM worker -WHERE status = 'active'; -``` - -## Performance - -### Image Sizes - -| Image | Size | Build Time (Cold) | Build Time (Cached) | -|-------|------|-------------------|---------------------| -| worker-base | ~580 MB | ~5 min | ~30 sec | -| worker-python | ~1.2 GB | ~6 min | ~45 sec | -| worker-node | ~760 MB | ~6 min | ~45 sec | -| worker-full | ~1.6 GB | ~7 min | ~60 sec | - -### Optimization Tips - -1. **Use specific variants**: Don't use `worker-full` if you only need Python -2. **Enable BuildKit**: Dramatically speeds up builds -3. **Layer caching**: Order Dockerfile commands from least to most frequently changed -4. **Multi-stage builds**: Keeps runtime images small - -## Files - -- `Dockerfile.worker.optimized` - Multi-stage worker Dockerfile with all variants -- `README.worker.md` - This file -- `../docker-compose.yaml` - Service definitions for all workers - -## References - -- [Worker Containerization Design](../docs/worker-containerization.md) -- [Quick Start Guide](../docs/worker-containers-quickstart.md) -- [Worker Service Architecture](../docs/architecture/worker-service.md) -- [Production Deployment](../docs/production-deployment.md) - -## Quick Commands - -```bash -# Build all workers -make docker-build-workers - -# Start all workers -docker-compose up -d worker-shell worker-python worker-node worker-full - -# Check worker status -docker-compose exec postgres psql -U attune -d attune -c \ - "SELECT name, status, capabilities FROM worker;" - -# View Python worker logs -docker-compose logs -f worker-python - -# Restart worker -docker-compose restart worker-python - -# Scale Python workers -docker-compose up -d --scale worker-python=3 - -# Stop all workers -docker-compose stop worker-shell worker-python worker-node worker-full -```