"""Bacalhau REST client connector and Blossom CAS compute-over-data mounting bridge.
Implements Phase 2 of the compute roadmap:
- BacalhauClientProtocol: abstract interface for Bacalhau daemon orchestrators.
- BacalhauRestClient: native HTTP/REST connector for Bacalhau v1 orchestrator API.
- MockBacalhauClient: test double for deterministic testing and dry-run fixtures.
- BlossomMountBridge: Blossom CAS input resolution and output packaging/uploading.
"""
from __future__ import annotations
import hashlib
import json
import os
from typing import Any, Callable, Mapping, NamedTuple, Optional, Protocol, Sequence, TypedDict
from fleetmesh_node.transport import netfetch
from ..common import ComputeError, resolve_blob
from ..sandbox import ComputeLimits
from ..types import BacalhauJobSummary
from .base import ComputeExecutionResult, ComputeExecutor, ComputeQuote
ProgressCallback = Callable[[str, float, Mapping[str, Any]], None]
[docs]class BlossomStagingResult(NamedTuple):
"""Result of staging Blossom CAS input blobs into container mount points."""
staged_paths: list[str]
total_bytes: int
files: dict[str, bytes]
source_cids: list[str]
[docs]class BlossomUploadResult(NamedTuple):
"""Result of packaging and committing container outputs to Blossom CAS."""
result_cid: str
result_hash: str
output_bytes: bytes
mounted_outputs: list[str]
[docs]class BacalhauTaskSpec(TypedDict, total=False):
"""Specification of a single Bacalhau container task."""
name: str
engine: str
image: str
entrypoint: list[str]
parameters: list[str]
env: dict[str, str]
input_sources: list[Mapping[str, Any]]
result_paths: list[Mapping[str, Any]]
resources: Mapping[str, Any]
timeouts: dict[str, int]
[docs]class BacalhauClientProtocol(Protocol):
"""Interface required for Bacalhau orchestrator connectors."""
[docs] def submit_job(self, job_payload: Mapping[str, Any], timeout_seconds: int = 300) -> str:
"""Submit a batch job specification and return assigned JobID."""
...
[docs] def get_job_status(self, job_id: str) -> Mapping[str, Any]:
"""Query execution lifecycle state for the given job_id."""
...
[docs] def get_job_results(self, job_id: str) -> Mapping[str, Any]:
"""Fetch outputs, logs, and artifacts for a completed job."""
...
[docs] def cancel_job(self, job_id: str, reason: str = '') -> bool:
"""Abort or terminate a running Bacalhau job."""
...
[docs] def is_alive(self) -> bool:
"""Check if the Bacalhau daemon/orchestrator endpoint is reachable."""
...
[docs]class BacalhauRestClient:
"""Live Bacalhau REST API v1 connector using Fleetmesh netfetch."""
[docs] def __init__(
self,
endpoint: str = 'http://127.0.0.1:1234',
timeout_seconds: float = 30.0,
) -> None:
self.endpoint: str = endpoint.rstrip('/')
self.timeout_seconds: float = timeout_seconds
[docs] def is_alive(self) -> bool:
"""Check if the Bacalhau daemon endpoint responds to health probes."""
for api_route in ('/api/v1/orchestrator/nodes', '/api/v1/version', '/api/v1/agent/alive'):
url = f"{self.endpoint}{api_route}"
try:
body, _ = netfetch.fetch(
url,
max_bytes=65536,
timeout=2.0,
method='GET',
allow_private=True,
)
if body:
return True
except Exception:
continue
return False
[docs] def build_bacalhau_job_spec(self, job_payload: Mapping[str, Any]) -> Mapping[str, Any]:
"""Convert Fleetmesh job descriptor into standard Bacalhau v1 format."""
engine = str(job_payload.get('engine', 'docker')).lower()
image = str(job_payload.get('image', ''))
wasm_mod = job_payload.get('wasm_module', {})
entrypoint = job_payload.get('entrypoint') or ['/bin/sh', '-c']
parameters = job_payload.get('parameters') or []
env_vars = job_payload.get('env') or {}
resources = job_payload.get('resources') or {'cpu': 1, 'gpu': 0, 'memory_gib': 2.0, 'timeout_seconds': 300}
storage_mounts = job_payload.get('storage_mounts') or []
output_mounts = job_payload.get('output_mounts') or []
input_sources: list[Mapping[str, Any]] = []
for mount in storage_mounts:
cid = mount.get('source_cid', '')
mount_path = mount.get('mount_path', '/inputs')
input_sources.append({
'Source': {
'Type': 'urlDownload' if cid.startswith('http') else 's3' if cid.startswith('s3://') else 'localDirectory',
'Params': {
'URL': cid if cid.startswith('http') else '',
'Target': mount_path,
}
}
})
result_paths = [
{'Name': m.get('name', 'output'), 'Path': m.get('container_path', '/outputs')}
for m in output_mounts
] or [{'Name': 'outputs', 'Path': '/outputs'}]
task = {
'Name': 'main',
'Engine': {
'Type': engine,
'Params': {
'Image': image or wasm_mod.get('sha256', ''),
'Entrypoint': entrypoint,
'Parameters': parameters,
'EnvironmentVariables': [f"{k}={v}" for k, v in env_vars.items()],
}
},
'InputSources': input_sources,
'ResultPaths': result_paths,
'Resources': {
'CPU': str(resources.get('cpu', 1)),
'Memory': f"{int(resources.get('memory_gib', 2))}Gb",
'GPU': str(resources.get('gpu', 0)),
},
'Timeouts': {
'ExecutionTimeout': int(resources.get('timeout_seconds', 300)),
},
'Network': {
'Type': 'None',
},
}
job_id = str(job_payload.get('id') or os.urandom(8).hex())
return {
'Job': {
'Name': f"fleetmesh-{job_id[:16]}",
'Type': 'batch',
'Count': 1,
'Tasks': [task],
}
}
[docs] def submit_job(self, job_payload: Mapping[str, Any], timeout_seconds: int = 300) -> str:
"""Submit a job specification to the Bacalhau orchestrator."""
spec = self.build_bacalhau_job_spec(job_payload)
data_bytes = json.dumps(spec).encode('utf-8')
# Try orchestrator API first, fall back to /api/v1/run
last_err: Optional[Exception] = None
for api_route in ('/api/v1/orchestrator/jobs', '/api/v1/run'):
url = f"{self.endpoint}{api_route}"
try:
body, _ = netfetch.fetch(
url,
max_bytes=1048576,
timeout=self.timeout_seconds,
method='POST',
data=data_bytes,
headers={'Content-Type': 'application/json', 'User-Agent': 'fleetmesh-bacalhau/1.0'},
allow_private=True,
)
resp = json.loads(body.decode('utf-8'))
job_id = resp.get('JobID') or resp.get('ID') or (resp.get('Job') or {}).get('ID')
if job_id:
return str(job_id)
except netfetch.FetchError as e:
last_err = e
continue
except Exception as e:
last_err = e
continue
raise ComputeError(f"Bacalhau job submission failed: {last_err}")
[docs] def get_job_status(self, job_id: str) -> Mapping[str, Any]:
"""Fetch current state of a Bacalhau job."""
url = f"{self.endpoint}/api/v1/orchestrator/jobs/{job_id}"
try:
body, _ = netfetch.fetch(
url,
max_bytes=524288,
timeout=self.timeout_seconds,
method='GET',
allow_private=True,
)
data = json.loads(body.decode('utf-8'))
state = data.get('State') or (data.get('Job') or {}).get('State') or {}
status_str = state.get('StateType') or 'Running'
return {
'job_id': job_id,
'state': status_str,
'raw': data,
}
except Exception as e:
raise ComputeError(f"Failed to fetch Bacalhau job status for {job_id}: {e}")
[docs] def get_job_results(self, job_id: str) -> Mapping[str, Any]:
"""Fetch results and output references for a completed job."""
url = f"{self.endpoint}/api/v1/orchestrator/jobs/{job_id}/results"
try:
body, _ = netfetch.fetch(
url,
max_bytes=1048576,
timeout=self.timeout_seconds,
method='GET',
allow_private=True,
)
return json.loads(body.decode('utf-8'))
except Exception as e:
raise ComputeError(f"Failed to fetch Bacalhau job results for {job_id}: {e}")
[docs] def cancel_job(self, job_id: str, reason: str = '') -> bool:
"""Cancel an in-flight Bacalhau job."""
url = f"{self.endpoint}/api/v1/orchestrator/jobs/{job_id}/stop"
try:
data_bytes = json.dumps({'Reason': reason or 'cancelled by fleetmesh'}).encode('utf-8')
netfetch.fetch(
url,
max_bytes=65536,
timeout=self.timeout_seconds,
method='POST',
data=data_bytes,
headers={'Content-Type': 'application/json'},
allow_private=True,
)
return True
except Exception:
return False
[docs]class MockBacalhauClient:
"""Test double implementing BacalhauClientProtocol for isolated and mock tests."""
[docs] def __init__(self, simulate_failure: bool = False) -> None:
self.submitted_jobs: list[Mapping[str, Any]] = []
self.cancelled_jobs: list[str] = []
self.simulate_failure: bool = simulate_failure
self.call_history: list[str] = []
[docs] def is_alive(self) -> bool:
return True
[docs] def submit_job(self, job_payload: Mapping[str, Any], timeout_seconds: int = 300) -> str:
self.call_history.append('submit_job')
if self.simulate_failure:
raise ComputeError("Simulated Bacalhau daemon failure")
job_id = f"mock-bacalhau-{os.urandom(8).hex()}"
self.submitted_jobs.append({'job_id': job_id, 'payload': job_payload})
return job_id
[docs] def get_job_status(self, job_id: str) -> Mapping[str, Any]:
self.call_history.append(f'get_job_status:{job_id}')
if job_id in self.cancelled_jobs:
return {'job_id': job_id, 'state': 'Cancelled'}
return {'job_id': job_id, 'state': 'Completed'}
[docs] def get_job_results(self, job_id: str) -> Mapping[str, Any]:
self.call_history.append(f'get_job_results:{job_id}')
return {
'job_id': job_id,
'status': 'success',
'output_cid': f"bafkrei{hashlib.sha256(job_id.encode()).hexdigest()[:48]}",
'exit_code': 0,
}
[docs] def cancel_job(self, job_id: str, reason: str = '') -> bool:
self.call_history.append(f'cancel_job:{job_id}')
self.cancelled_jobs.append(job_id)
return True
[docs]class BlossomMountBridge:
"""Manages input staging from Blossom CAS and output artifact publishing."""
[docs] def package_and_upload_outputs(
self,
raw_output_or_files: bytes | Mapping[str, bytes],
output_mounts: Sequence[Mapping[str, Any]],
blob_store: Optional[Any] = None,
blossom_servers: Optional[Sequence[str]] = None,
) -> BlossomUploadResult:
"""Hash, persist, and register job output artifacts into Blossom CAS."""
if isinstance(raw_output_or_files, bytes):
out_bytes = raw_output_or_files
else:
out_bytes = json.dumps(
{k: v.decode('utf-8', errors='replace') for k, v in raw_output_or_files.items()},
sort_keys=True
).encode('utf-8')
result_hash = hashlib.sha256(out_bytes).hexdigest()
# Persist to local blob_store if present
result_cid = None
if blob_store is not None and hasattr(blob_store, 'put'):
try:
desc = blob_store.put(out_bytes)
result_cid = desc.get('cid') if isinstance(desc, dict) else str(desc)
except Exception:
pass
if not result_cid:
# Format canonical Blossom SHA-256 CID
result_cid = result_hash
# Push to remote Blossom servers if configured
if blossom_servers:
for srv in blossom_servers:
try:
url = f"{srv.rstrip('/')}/upload"
netfetch.fetch(
url,
max_bytes=1048576,
timeout=15.0,
method='PUT',
data=out_bytes,
headers={'Content-Type': 'application/octet-stream', 'X-SHA-256': result_hash},
allow_private=True,
)
except Exception:
pass
mounted_outputs = [str(m.get('name', 'outputs')) for m in output_mounts] or ['default']
return BlossomUploadResult(
result_cid=result_cid,
result_hash=result_hash,
output_bytes=out_bytes,
mounted_outputs=mounted_outputs,
)
[docs]class BacalhauExecutor(ComputeExecutor):
"""Compute-over-data batch driver backed by Bacalhau / Docker containers.
Integrates Phase 2 capabilities:
- Mounts Blossom CAS CIDs directly into container volume mounts.
- Dispatches jobs to Bacalhau orchestrator or local sandbox.
- Emits 5-state lifecycle progress heartbeats (compute/1.1/progress).
- Honors cooperative cancellation tokens (compute/1.1/cancel).
"""
engine_id: str = 'bacalhau'
wasi_profile_digest: str = ''
[docs] def __init__(
self,
endpoint: str = 'http://127.0.0.1:1234',
mock_mode: bool = False,
client: Optional[BacalhauClientProtocol] = None,
mount_bridge: Optional[BlossomMountBridge] = None,
) -> None:
self.endpoint = endpoint.rstrip('/')
self.mock_mode = mock_mode
self.client = client
self.mount_bridge = mount_bridge or BlossomMountBridge()
[docs] def is_live_runnable(self) -> bool:
"""Check whether Bacalhau execution can succeed on this node."""
if self.mock_mode:
return True
if self.client is not None:
try:
return self.client.is_alive()
except Exception:
return False
return False
[docs] def quote(
self,
module_spec: Mapping[str, Any],
input_spec: Mapping[str, Any],
limits: ComputeLimits,
) -> ComputeQuote:
cpu_minutes = int(input_spec.get('cpu_minutes', 1))
gpu_minutes = int(input_spec.get('gpu_minutes', 0))
ram_gib = float(input_spec.get('ram_gib', 1.0))
fuel_est = int(cpu_minutes * 150 + gpu_minutes * 1200 + ram_gib * 50)
return ComputeQuote(
fuel_estimate=min(fuel_est, limits.max_fuel),
rate_unit='msat_per_job',
rate_amount=fuel_est,
engine_id=self.engine_id,
)
[docs] def execute(
self,
job_payload: Mapping[str, Any],
fuel_cap: int,
progress_callback: Optional[ProgressCallback] = None,
cancellation_token: Optional[Any] = None,
) -> ComputeExecutionResult:
engine = job_payload.get('engine', 'docker')
image = job_payload.get('image', '')
wasm_module = job_payload.get('wasm_module', {})
entrypoint = job_payload.get('entrypoint') or ['/bin/sh', '-c']
parameters = job_payload.get('parameters') or []
storage_mounts = job_payload.get('storage_mounts') or []
output_mounts = job_payload.get('output_mounts') or []
resources = job_payload.get('resources') or {'cpu': 1, 'gpu': 0, 'memory_gib': 2.0, 'timeout_seconds': 60}
verif_mode = self.verification_mode(job_payload)
# Honesty gate. If not in mock_mode, an active and verified live backend must be present.
if not self.mock_mode and self.client is None and not self.is_live_runnable():
raise ComputeError('not implemented: bacalhau executor is a stub (mock_mode only)')
def _check_cancelled() -> None:
if cancellation_token is not None:
is_set = False
if hasattr(cancellation_token, 'is_set') and cancellation_token.is_set():
is_set = True
elif isinstance(cancellation_token, bool) and cancellation_token:
is_set = True
if is_set:
raise ComputeError('job cancelled by requester')
# 1. State: queued
if progress_callback:
progress_callback('queued', 0.0, {'status': 'job queued for batch execution'})
# 2. State: downloading_inputs (stage Blossom CAS volumes)
_check_cancelled()
if progress_callback:
progress_callback('downloading_inputs', 0.2, {'status': 'staging CAS input volumes'})
blob_store = job_payload.get('blob_store')
blossom_servers = job_payload.get('blossom_servers')
staging_res = self.mount_bridge.resolve_and_stage_inputs(
storage_mounts=storage_mounts,
blob_store=blob_store,
blossom_servers=blossom_servers,
)
mounted_inputs = staging_res.files
total_in_bytes = staging_res.total_bytes
# 3. State: running (execute via Bacalhau orchestrator or deterministic pricing model)
_check_cancelled()
if progress_callback:
progress_callback('running', 0.5, {'status': 'executing container workload'})
client_results: Optional[Mapping[str, Any]] = None
if self.client is not None:
timeout_sec = int(resources.get('timeout_seconds') or 60)
job_id = self.client.submit_job(job_payload, timeout_seconds=timeout_sec)
try:
_check_cancelled()
status_data = self.client.get_job_status(job_id)
state_str = str(status_data.get('state', '')).lower()
if 'cancel' in state_str:
raise ComputeError('job cancelled by requester')
if 'fail' in state_str or 'error' in state_str:
raise ComputeError(f"Bacalhau job failed: {status_data}")
_check_cancelled()
client_results = self.client.get_job_results(job_id)
except ComputeError:
raise
except Exception as e:
if cancellation_token is not None and getattr(cancellation_token, 'is_set', lambda: False)():
self.client.cancel_job(job_id, 'cancelled by fleetmesh session')
raise ComputeError('job cancelled by requester')
raise ComputeError(f"Bacalhau job execution failed: {e}")
cpu = int(resources.get('cpu') or 1)
gpu = int(resources.get('gpu') or 0)
timeout_sec = int(resources.get('timeout_seconds') or 60)
cpu_sec = min(timeout_sec, 10)
gpu_sec = min(timeout_sec, 10) if gpu > 0 else 0
mem_gib = float(resources.get('memory_gib') or 1.0)
fuel_used = cpu * 150 + gpu * 1200 + int(mem_gib * 50)
if fuel_used > fuel_cap:
raise ComputeError(f"Bacalhau job exceeded fuel cap ({fuel_used} > {fuel_cap})")
# 4. State: uploading_outputs (package artifacts and commit to Blossom CAS)
_check_cancelled()
if progress_callback:
progress_callback('uploading_outputs', 0.8, {'status': 'packaging and uploading output artifacts'})
out_summary: BacalhauJobSummary = {
'status': 'completed',
'engine': engine,
'image': image or wasm_module.get('sha256', ''),
'entrypoint': entrypoint,
'parameters': parameters,
'mounted_paths': list(mounted_inputs.keys()),
'outputs': [m.get('name', 'output') for m in output_mounts] or ['default'],
'bytes_processed': total_in_bytes,
}
if client_results:
out_summary['results'] = client_results
upload_res = self.mount_bridge.package_and_upload_outputs(
raw_output_or_files=json.dumps(out_summary, sort_keys=True).encode('utf-8'),
output_mounts=output_mounts,
blob_store=blob_store,
blossom_servers=blossom_servers,
)
raw_output = upload_res.output_bytes
result_hash = upload_res.result_hash
# 5. State: completed
if progress_callback:
progress_callback('completed', 1.0, {
'status': 'job completed successfully',
'result_hash': result_hash,
'result_cid': upload_res.result_cid,
})
counters = {
'cpu_seconds': cpu_sec,
'gpu_seconds': gpu_sec,
'bytes_in': total_in_bytes,
'bytes_out': len(raw_output),
}
return ComputeExecutionResult(
result_hash=result_hash,
raw_output=raw_output,
fuel_used=fuel_used,
engine_id=self.engine_id,
wasi_profile_digest=f"bacalhau:{engine}:{image or 'wasm'}",
counters=counters,
verification_mode=verif_mode,
)
__all__ = [
'ProgressCallback',
'BlossomStagingResult',
'BlossomUploadResult',
'BacalhauTaskSpec',
'BacalhauClientProtocol',
'BacalhauRestClient',
'MockBacalhauClient',
'BlossomMountBridge',
'BacalhauExecutor',
]