attune-system/attune
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

working on runtime executions

Browse Source
This commit is contained in:
David Culbreth
2026-02-16 22:04:20 -06:00
parent f52320f889
commit 904ede04be
99 changed files with 6778 additions and 5929 deletions
Download Patch File Download Diff File Expand all files Collapse all files

184
scripts/load_core_pack.py
View File

@@ -212,7 +212,109 @@ class PackLoader:
cursor.close()
return trigger_ids
def upsert_actions(self) -> Dict[str, int]:
def upsert_runtimes(self) -> Dict[str, int]:
"""Load runtime definitions from runtimes/*.yaml"""
print("\n→ Loading runtimes...")
runtimes_dir = self.pack_dir / "runtimes"
if not runtimes_dir.exists():
print(" No runtimes directory found")
return {}
runtime_ids = {}
cursor = self.conn.cursor()
for yaml_file in sorted(runtimes_dir.glob("*.yaml")):
runtime_data = self.load_yaml(yaml_file)
if not runtime_data:
continue
ref = runtime_data.get("ref")
if not ref:
print(
f" ⚠ Runtime YAML {yaml_file.name} missing 'ref' field, skipping"
)
continue
name = runtime_data.get("name", ref.split(".")[-1])
description = runtime_data.get("description", "")
distributions = json.dumps(runtime_data.get("distributions", {}))
installation = json.dumps(runtime_data.get("installation", {}))
execution_config = json.dumps(runtime_data.get("execution_config", {}))
cursor.execute(
"""
INSERT INTO runtime (
ref, pack, pack_ref, name, description,
distributions, installation, execution_config
)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (ref) DO UPDATE SET
name = EXCLUDED.name,
description = EXCLUDED.description,
distributions = EXCLUDED.distributions,
installation = EXCLUDED.installation,
execution_config = EXCLUDED.execution_config,
updated = NOW()
RETURNING id
""",
(
ref,
self.pack_id,
self.pack_ref,
name,
description,
distributions,
installation,
execution_config,
),
)
runtime_id = cursor.fetchone()[0]
runtime_ids[ref] = runtime_id
# Also index by lowercase name for easy lookup by runner_type
runtime_ids[name.lower()] = runtime_id
print(f" ✓ Runtime '{ref}' (ID: {runtime_id})")
cursor.close()
return runtime_ids
def resolve_action_runtime(
self, action_data: Dict, runtime_ids: Dict[str, int]
) -> Optional[int]:
"""Resolve the runtime ID for an action based on runner_type or entrypoint."""
runner_type = action_data.get("runner_type", "").lower()
if not runner_type:
# Try to infer from entrypoint extension
entrypoint = action_data.get("entry_point", "")
if entrypoint.endswith(".py"):
runner_type = "python"
elif entrypoint.endswith(".js"):
runner_type = "node.js"
else:
runner_type = "shell"
# Map runner_type names to runtime refs/names
lookup_keys = {
"shell": ["shell", "core.shell"],
"python": ["python", "core.python"],
"python3": ["python", "core.python"],
"node": ["node.js", "nodejs", "core.nodejs"],
"nodejs": ["node.js", "nodejs", "core.nodejs"],
"node.js": ["node.js", "nodejs", "core.nodejs"],
"native": ["native", "core.native"],
}
keys_to_try = lookup_keys.get(runner_type, [runner_type])
for key in keys_to_try:
if key in runtime_ids:
return runtime_ids[key]
print(f" ⚠ Could not resolve runtime for runner_type '{runner_type}'")
return None
def upsert_actions(self, runtime_ids: Dict[str, int]) -> Dict[str, int]:
"""Load action definitions"""
print("\n→ Loading actions...")
@@ -224,9 +326,6 @@ class PackLoader:
action_ids = {}
cursor = self.conn.cursor()
# First, ensure we have a runtime for actions
runtime_id = self.ensure_shell_runtime(cursor)
for yaml_file in sorted(actions_dir.glob("*.yaml")):
action_data = self.load_yaml(yaml_file)
@@ -251,6 +350,9 @@ class PackLoader:
entrypoint = str(script_path.relative_to(self.packs_dir))
break
# Resolve runtime ID for this action
runtime_id = self.resolve_action_runtime(action_data, runtime_ids)
param_schema = json.dumps(action_data.get("parameters", {}))
out_schema = json.dumps(action_data.get("output", {}))
@@ -326,32 +428,9 @@ class PackLoader:
cursor.close()
return action_ids
def ensure_shell_runtime(self, cursor) -> int:
"""Ensure shell runtime exists"""
cursor.execute(
"""
INSERT INTO runtime (
ref, pack, pack_ref, name, description, distributions
)
VALUES (%s, %s, %s, %s, %s, %s)
ON CONFLICT (ref) DO UPDATE SET
name = EXCLUDED.name,
description = EXCLUDED.description,
updated = NOW()
RETURNING id
""",
(
"core.action.shell",
self.pack_id,
self.pack_ref,
"Shell",
"Shell script runtime",
json.dumps({"shell": {"command": "sh"}}),
),
)
return cursor.fetchone()[0]
def upsert_sensors(self, trigger_ids: Dict[str, int]) -> Dict[str, int]:
def upsert_sensors(
self, trigger_ids: Dict[str, int], runtime_ids: Dict[str, int]
) -> Dict[str, int]:
"""Load sensor definitions"""
print("\n→ Loading sensors...")
@@ -363,8 +442,12 @@ class PackLoader:
sensor_ids = {}
cursor = self.conn.cursor()
# Ensure sensor runtime exists
sensor_runtime_id = self.ensure_sensor_runtime(cursor)
# Look up sensor runtime from already-loaded runtimes
sensor_runtime_id = runtime_ids.get("builtin") or runtime_ids.get(
"core.builtin"
)
if not sensor_runtime_id:
print(" ⚠ No sensor runtime found, sensors will have no runtime")
for yaml_file in sorted(sensors_dir.glob("*.yaml")):
sensor_data = self.load_yaml(yaml_file)
@@ -438,7 +521,7 @@ class PackLoader:
description,
entry_point,
sensor_runtime_id,
"core.sensor.builtin",
"core.builtin",
trigger_id,
trigger_ref,
enabled,
@@ -453,31 +536,6 @@ class PackLoader:
cursor.close()
return sensor_ids
def ensure_sensor_runtime(self, cursor) -> int:
"""Ensure sensor runtime exists"""
cursor.execute(
"""
INSERT INTO runtime (
ref, pack, pack_ref, name, description, distributions
)
VALUES (%s, %s, %s, %s, %s, %s)
ON CONFLICT (ref) DO UPDATE SET
name = EXCLUDED.name,
description = EXCLUDED.description,
updated = NOW()
RETURNING id
""",
(
"core.sensor.builtin",
self.pack_id,
self.pack_ref,
"Built-in Sensor",
"Built-in sensor runtime",
json.dumps([]),
),
)
return cursor.fetchone()[0]
def load_pack(self):
"""Main loading process"""
print("=" * 60)
@@ -493,14 +551,17 @@ class PackLoader:
# Load pack metadata
self.upsert_pack()
# Load runtimes first (actions and sensors depend on them)
runtime_ids = self.upsert_runtimes()
# Load triggers
trigger_ids = self.upsert_triggers()
# Load actions
action_ids = self.upsert_actions()
# Load actions (with runtime resolution)
action_ids = self.upsert_actions(runtime_ids)
# Load sensors
sensor_ids = self.upsert_sensors(trigger_ids)
sensor_ids = self.upsert_sensors(trigger_ids, runtime_ids)
# Commit all changes
self.conn.commit()
@@ -509,6 +570,7 @@ class PackLoader:
print(f"✓ Pack '{self.pack_name}' loaded successfully!")
print("=" * 60)
print(f" Pack ID: {self.pack_id}")
print(f" Runtimes: {len(set(runtime_ids.values()))}")
print(f" Triggers: {len(trigger_ids)}")
print(f" Actions: {len(action_ids)}")
print(f" Sensors: {len(sensor_ids)}")

28
scripts/seed_core_pack.sql
View File

@@ -32,16 +32,15 @@ BEGIN
-- Get core pack ID
SELECT id INTO v_pack_id FROM attune.pack WHERE ref = 'core';
-- Create shell runtime for actions
INSERT INTO attune.runtime (ref, pack, pack_ref, name, description, runtime_type, distributions)
-- Create shell runtime
INSERT INTO attune.runtime (ref, pack, pack_ref, name, description, distributions)
VALUES (
'core.action.shell',
'core.shell',
v_pack_id,
'core',
'shell',
'Execute shell commands',
'action',
'{"shell": {"command": "sh"}}'::jsonb
'Shell',
'Shell (bash/sh) runtime for script execution - always available',
'{"verification": {"always_available": true}}'::jsonb
)
ON CONFLICT (ref) DO UPDATE SET
name = EXCLUDED.name,
@@ -49,16 +48,15 @@ BEGIN
updated = NOW()
RETURNING id INTO v_action_runtime_id;
-- Create built-in runtime for sensors
INSERT INTO attune.runtime (ref, pack, pack_ref, name, description, runtime_type, distributions)
-- Create built-in runtime for sensors (no execution_config = not executable by worker)
INSERT INTO attune.runtime (ref, pack, pack_ref, name, description, distributions)
VALUES (
'core.sensor.builtin',
'core.builtin',
v_pack_id,
'core',
'Built-in',
'Built-in runtime for system timers and sensors',
'sensor',
'[]'::jsonb
'Builtin',
'Built-in sensor runtime for native Attune sensors (timers, webhooks, etc.)',
'{"verification": {"always_available": true, "check_required": false}, "type": "builtin"}'::jsonb
)
ON CONFLICT (ref) DO UPDATE SET
name = EXCLUDED.name,
@@ -370,7 +368,7 @@ BEGIN
'Timer sensor that fires every 10 seconds',
'builtin:interval_timer',
v_sensor_runtime_id,
'core.sensor.builtin',
'core.builtin',
v_intervaltimer_id,
'core.intervaltimer',
true,

311
scripts/seed_runtimes.sql
View File

@@ -1,229 +1,238 @@
-- Seed Default Runtimes
-- Description: Inserts default runtime configurations for actions and sensors
-- Description: Inserts default runtime configurations for the core pack
-- This should be run after migrations to populate the runtime table with core runtimes
--
-- Runtimes are unified (no action/sensor distinction). Whether a runtime can
-- execute actions is determined by the presence of an execution_config with an
-- interpreter. The builtin runtime has no execution_config and is used only for
-- internal sensors (timers, webhooks, etc.).
--
-- The execution_config JSONB column drives how the worker executes actions and
-- how pack installation sets up environments. Template variables:
-- {pack_dir} - absolute path to the pack directory
-- {env_dir} - resolved environment directory (runtime_envs_dir/pack_ref/runtime_name)
-- {interpreter} - resolved interpreter path
-- {action_file} - absolute path to the action script file
-- {manifest_path} - absolute path to the dependency manifest file
SET search_path TO attune, public;
-- ============================================================================
-- ACTION RUNTIMES
-- UNIFIED RUNTIMES (5 total)
-- ============================================================================
-- Python 3 Action Runtime
INSERT INTO attune.runtime (
-- Python 3 Runtime
INSERT INTO runtime (
ref,
pack_ref,
name,
description,
runtime_type,
distributions,
installation
installation,
execution_config
) VALUES (
'core.action.python3',
'core.python',
'core',
'Python 3 Action Runtime',
'Execute actions using Python 3.x interpreter',
'action',
'["python3"]'::jsonb,
'Python',
'Python 3 runtime for actions and sensors with automatic environment management',
'{
"method": "system",
"package_manager": "pip",
"requirements_file": "requirements.txt"
"verification": {
"commands": [
{"binary": "python3", "args": ["--version"], "exit_code": 0, "pattern": "Python 3\\\\.", "priority": 1},
{"binary": "python", "args": ["--version"], "exit_code": 0, "pattern": "Python 3\\\\.", "priority": 2}
]
},
"min_version": "3.8",
"recommended_version": "3.11"
}'::jsonb,
'{
"package_managers": ["pip", "pipenv", "poetry"],
"virtual_env_support": true
}'::jsonb,
'{
"interpreter": {
"binary": "python3",
"args": ["-u"],
"file_extension": ".py"
},
"environment": {
"env_type": "virtualenv",
"dir_name": ".venv",
"create_command": ["python3", "-m", "venv", "{env_dir}"],
"interpreter_path": "{env_dir}/bin/python3"
},
"dependencies": {
"manifest_file": "requirements.txt",
"install_command": ["{interpreter}", "-m", "pip", "install", "-r", "{manifest_path}"]
}
}'::jsonb
) ON CONFLICT (ref) DO UPDATE SET
name = EXCLUDED.name,
description = EXCLUDED.description,
distributions = EXCLUDED.distributions,
installation = EXCLUDED.installation,
execution_config = EXCLUDED.execution_config,
updated = NOW();
-- Shell Action Runtime
INSERT INTO attune.runtime (
-- Shell Runtime
INSERT INTO runtime (
ref,
pack_ref,
name,
description,
runtime_type,
distributions,
installation
installation,
execution_config
) VALUES (
'core.action.shell',
'core.shell',
'core',
'Shell Action Runtime',
'Execute actions using system shell (bash/sh)',
'action',
'["bash", "sh"]'::jsonb,
'Shell',
'Shell (bash/sh) runtime for script execution - always available',
'{
"method": "system",
"shell": "/bin/bash"
"verification": {
"commands": [
{"binary": "sh", "args": ["--version"], "exit_code": 0, "optional": true, "priority": 1},
{"binary": "bash", "args": ["--version"], "exit_code": 0, "optional": true, "priority": 2}
],
"always_available": true
}
}'::jsonb,
'{
"interpreters": ["sh", "bash", "dash"],
"portable": true
}'::jsonb,
'{
"interpreter": {
"binary": "/bin/bash",
"args": [],
"file_extension": ".sh"
}
}'::jsonb
) ON CONFLICT (ref) DO UPDATE SET
name = EXCLUDED.name,
description = EXCLUDED.description,
distributions = EXCLUDED.distributions,
installation = EXCLUDED.installation,
execution_config = EXCLUDED.execution_config,
updated = NOW();
-- Node.js Action Runtime
INSERT INTO attune.runtime (
-- Node.js Runtime
INSERT INTO runtime (
ref,
pack_ref,
name,
description,
runtime_type,
distributions,
installation
installation,
execution_config
) VALUES (
'core.action.nodejs',
'core.nodejs',
'core',
'Node.js Action Runtime',
'Execute actions using Node.js runtime',
'action',
'["nodejs", "node"]'::jsonb,
'Node.js',
'Node.js runtime for JavaScript-based actions and sensors',
'{
"method": "system",
"package_manager": "npm",
"requirements_file": "package.json"
"verification": {
"commands": [
{"binary": "node", "args": ["--version"], "exit_code": 0, "pattern": "v\\\\d+\\\\.\\\\d+\\\\.\\\\d+", "priority": 1}
]
},
"min_version": "16.0.0",
"recommended_version": "20.0.0"
}'::jsonb,
'{
"package_managers": ["npm", "yarn", "pnpm"],
"module_support": true
}'::jsonb,
'{
"interpreter": {
"binary": "node",
"args": [],
"file_extension": ".js"
},
"environment": {
"env_type": "node_modules",
"dir_name": "node_modules",
"create_command": ["npm", "init", "-y"],
"interpreter_path": null
},
"dependencies": {
"manifest_file": "package.json",
"install_command": ["npm", "install", "--prefix", "{pack_dir}"]
}
}'::jsonb
) ON CONFLICT (ref) DO UPDATE SET
name = EXCLUDED.name,
description = EXCLUDED.description,
distributions = EXCLUDED.distributions,
installation = EXCLUDED.installation,
execution_config = EXCLUDED.execution_config,
updated = NOW();
-- Native Action Runtime (for compiled Rust binaries and other native executables)
INSERT INTO attune.runtime (
-- Native Runtime (for compiled binaries: Rust, Go, C, etc.)
INSERT INTO runtime (
ref,
pack_ref,
name,
description,
runtime_type,
distributions,
installation
installation,
execution_config
) VALUES (
'core.action.native',
'core.native',
'core',
'Native Action Runtime',
'Execute actions as native compiled binaries',
'action',
'["native"]'::jsonb,
'Native',
'Native compiled runtime (Rust, Go, C, etc.) - always available',
'{
"method": "binary",
"description": "Native executable - no runtime installation required"
"verification": {
"always_available": true,
"check_required": false
},
"languages": ["rust", "go", "c", "c++"]
}'::jsonb,
'{
"build_required": false,
"system_native": true
}'::jsonb,
'{
"interpreter": {
"binary": "/bin/sh",
"args": ["-c"],
"file_extension": null
}
}'::jsonb
) ON CONFLICT (ref) DO UPDATE SET
name = EXCLUDED.name,
description = EXCLUDED.description,
distributions = EXCLUDED.distributions,
installation = EXCLUDED.installation,
execution_config = EXCLUDED.execution_config,
updated = NOW();
-- ============================================================================
-- SENSOR RUNTIMES
-- ============================================================================
-- Python 3 Sensor Runtime
INSERT INTO attune.runtime (
-- Builtin Runtime (for internal sensors: timers, webhooks, etc.)
-- NOTE: No execution_config - this runtime cannot execute actions.
-- The worker skips runtimes without execution_config when loading.
INSERT INTO runtime (
ref,
pack_ref,
name,
description,
runtime_type,
distributions,
installation
) VALUES (
'core.sensor.python3',
'core.builtin',
'core',
'Python 3 Sensor Runtime',
'Execute sensors using Python 3.x interpreter',
'sensor',
'["python3"]'::jsonb,
'Builtin',
'Built-in sensor runtime for native Attune sensors (timers, webhooks, etc.)',
'{
"method": "system",
"package_manager": "pip",
"requirements_file": "requirements.txt"
}'::jsonb
) ON CONFLICT (ref) DO UPDATE SET
name = EXCLUDED.name,
description = EXCLUDED.description,
distributions = EXCLUDED.distributions,
installation = EXCLUDED.installation,
updated = NOW();
-- Shell Sensor Runtime
INSERT INTO attune.runtime (
ref,
pack_ref,
name,
description,
runtime_type,
distributions,
installation
) VALUES (
'core.sensor.shell',
'core',
'Shell Sensor Runtime',
'Execute sensors using system shell (bash/sh)',
'sensor',
'["bash", "sh"]'::jsonb,
"verification": {
"always_available": true,
"check_required": false
},
"type": "builtin"
}'::jsonb,
'{
"method": "system",
"shell": "/bin/bash"
}'::jsonb
) ON CONFLICT (ref) DO UPDATE SET
name = EXCLUDED.name,
description = EXCLUDED.description,
distributions = EXCLUDED.distributions,
installation = EXCLUDED.installation,
updated = NOW();
-- Node.js Sensor Runtime
INSERT INTO attune.runtime (
ref,
pack_ref,
name,
description,
runtime_type,
distributions,
installation
) VALUES (
'core.sensor.nodejs',
'core',
'Node.js Sensor Runtime',
'Execute sensors using Node.js runtime',
'sensor',
'["nodejs", "node"]'::jsonb,
'{
"method": "system",
"package_manager": "npm",
"requirements_file": "package.json"
}'::jsonb
) ON CONFLICT (ref) DO UPDATE SET
name = EXCLUDED.name,
description = EXCLUDED.description,
distributions = EXCLUDED.distributions,
installation = EXCLUDED.installation,
updated = NOW();
-- Native Sensor Runtime (for compiled Rust binaries and other native executables)
INSERT INTO attune.runtime (
ref,
pack_ref,
name,
description,
runtime_type,
distributions,
installation
) VALUES (
'core.sensor.native',
'core',
'Native Sensor Runtime',
'Execute sensors as native compiled binaries',
'sensor',
'["native"]'::jsonb,
'{
"method": "binary",
"description": "Native executable - no runtime installation required"
"method": "builtin",
"included_with_service": true
}'::jsonb
) ON CONFLICT (ref) DO UPDATE SET
name = EXCLUDED.name,
@@ -241,16 +250,16 @@ DO $$
DECLARE
runtime_count INTEGER;
BEGIN
SELECT COUNT(*) INTO runtime_count FROM attune.runtime WHERE pack_ref = 'core';
SELECT COUNT(*) INTO runtime_count FROM runtime WHERE pack_ref = 'core';
RAISE NOTICE 'Seeded % core runtime(s)', runtime_count;
END $$;
-- Show summary
SELECT
runtime_type,
COUNT(*) as count,
ARRAY_AGG(ref ORDER BY ref) as refs
FROM attune.runtime
ref,
name,
CASE WHEN execution_config IS NOT NULL AND execution_config != '{}'::jsonb
THEN 'yes' ELSE 'no' END AS executable
FROM runtime
WHERE pack_ref = 'core'
GROUP BY runtime_type
ORDER BY runtime_type;
ORDER BY ref;