"""Non-deterministic executors: LLM inference, Bacalhau jobs and agent runs.
These call out over HTTP (or run in mock mode) and are settled on evidence
rather than reproduction; see docs/non-deterministic-compute.md.
"""
from __future__ import annotations
import json
import time
from typing import Any, Mapping, Optional
from fleetmesh_core.crypto import crypto
from fleetmesh_node.transport import netfetch
from ..common import ComputeError, LLM_MAX_RESPONSE_BYTES
from ..sandbox import ComputeLimits
from .base import ComputeExecutionResult, ComputeExecutor, ComputeQuote
from .providers import (
PROVIDER_PRESETS,
LinkedProviderConfig,
OperatorProviderRouter,
ProviderPreset,
)
[docs]class LlmExecutor(ComputeExecutor):
"""Non-deterministic AI/LLM inference driver with grammar-guided decoding.
Supports OpenAI-compatible and Anthropic HTTP endpoints (OpenRouter, Claude,
OpenAI, Gemini, Copilot, Routstr, PayPerQuery, vLLM, Ollama, llama-server)
with CFG grammar constraints and fallback offline simulation.
"""
engine_id: str = 'llm'
wasi_profile_digest: str = ''
[docs] def __init__(
self,
endpoint: str = 'http://127.0.0.1:8000/v1',
default_model: str = 'meta-llama/Llama-3.3-70B-Instruct',
timeout_seconds: float = 30.0,
mock_mode: bool = False,
provider_router: Optional[OperatorProviderRouter] = None,
api_key: str = '',
cashu_token: str = '',
provider_id: str = 'vllm',
rate_msat_per_token: int = 12,
) -> None:
self.endpoint = endpoint.rstrip('/')
self.default_model = default_model
self.timeout_seconds = timeout_seconds
self.mock_mode = mock_mode
self.api_key = api_key
self.cashu_token = cashu_token
self.provider_id = provider_id
self.rate_msat_per_token = int(rate_msat_per_token)
self.provider_router: OperatorProviderRouter = provider_router or OperatorProviderRouter()
def _resolve_provider(self, job_payload: Optional[Mapping[str, Any]] = None) -> LinkedProviderConfig:
if job_payload is not None:
resolved = self.provider_router.resolve_for_job(job_payload)
if resolved is not None:
return resolved
else:
resolved = self.provider_router.get_provider()
if resolved is not None:
return resolved
return LinkedProviderConfig(
provider_id=self.provider_id,
label=self.provider_id.upper(),
endpoint=self.endpoint,
default_model=self.default_model,
models=(self.default_model,),
api_key=self.api_key,
cashu_token=self.cashu_token,
auth_scheme='bearer' if self.api_key else ('cashu' if self.cashu_token else 'none'),
wire_format='anthropic_messages' if self.provider_id in ('claude', 'anthropic') else 'openai',
rate_msat_per_token=self.rate_msat_per_token,
enabled=True,
)
[docs] def is_live_runnable(self) -> bool:
"""Check whether LLM inference can execute on this node."""
if self.mock_mode:
return True
if self.provider_router.runnable_providers():
return True
return bool(self.endpoint and self.endpoint.startswith(('http://', 'https://')))
[docs] def quote(
self,
module_spec: Mapping[str, Any],
input_spec: Mapping[str, Any],
limits: ComputeLimits,
) -> ComputeQuote:
merged_spec = dict(module_spec)
merged_spec.update(input_spec)
provider = self._resolve_provider(merged_spec)
rate = provider.rate_msat_per_token if provider else self.rate_msat_per_token
max_tokens = int(input_spec.get('max_tokens', limits.max_tokens or 2048))
prompt_est = len(str(input_spec.get('prompt', ''))) // 4
fuel_est = (prompt_est + max_tokens) * rate
return ComputeQuote(
fuel_estimate=min(fuel_est, limits.max_fuel),
rate_unit='msat_per_token',
rate_amount=rate,
engine_id=self.engine_id,
)
[docs] def execute(
self,
job_payload: Mapping[str, Any],
fuel_cap: int,
) -> ComputeExecutionResult:
provider = self._resolve_provider(job_payload)
model = job_payload.get('model') or provider.default_model or self.default_model
prompt = job_payload.get('prompt')
messages = job_payload.get('messages')
sampling = job_payload.get('sampling') or {}
grammar = job_payload.get('grammar') or {}
stream = bool(job_payload.get('stream', False))
# The backend endpoint is operator configuration only. An earlier
# version let ``job_payload['endpoint']`` override it, which handed a
# requester a POST-anywhere primitive against the node's own network
# (the default endpoint is loopback, so allow_private must be on).
verif_mode = self.verification_mode(job_payload)
if not messages and prompt is not None:
messages = [{'role': 'user', 'content': str(prompt)}]
elif not messages:
messages = [{'role': 'user', 'content': ''}]
max_tokens = int(sampling.get('max_tokens', 512))
temperature = float(sampling.get('temperature', 0.7))
top_p = float(sampling.get('top_p', 1.0))
seed = sampling.get('seed')
url, req_body = provider.format_request(
messages=messages,
model=model,
max_tokens=max_tokens,
temperature=temperature,
top_p=top_p,
seed=int(seed) if seed is not None else None,
grammar=grammar,
stream=stream,
)
raw_output_text: str = ''
prompt_tokens = max(1, sum(len(str(m.get('content', ''))) // 4 for m in messages))
completion_tokens = 0
gpu_seconds = 0.05
if self.mock_mode:
# Deterministic stand-in for tests and demos. It is only reachable
# when the operator constructed the executor with mock_mode=True;
# a live node never falls back to it.
if '_mock_completion' in job_payload and job_payload['_mock_completion'] is not None:
raw_output_text = str(job_payload['_mock_completion'])
elif grammar and (grammar.get('type') == 'json_schema' or 'json_schema' in grammar or grammar.get('response_format') == 'json_object'):
raw_output_text = json.dumps({
'status': 'completed',
'model': model,
'verified': True,
'result': 'synthesized-conforming-payload',
}, sort_keys=True)
else:
user_msg = messages[-1].get('content', '')
raw_output_text = f"Inference completion for [{model}]: {user_msg[:64]}"
completion_tokens = max(1, len(raw_output_text) // 4)
gpu_seconds = 0.025
else:
# A backend failure is a failed job, full stop.
data_bytes = json.dumps(req_body).encode('utf-8')
headers = provider.build_headers(
per_query_cashu_token=job_payload.get('_operator_cashu_token'),
)
start_t = time.time()
try:
body, _ = netfetch.fetch(
url,
max_bytes=LLM_MAX_RESPONSE_BYTES,
timeout=self.timeout_seconds,
method='POST',
data=data_bytes,
headers=headers,
# The endpoint is operator-configured (loopback vLLM/Ollama or
# operator-linked cloud/eCash provider), never requester-chosen.
allow_private=True,
)
resp_data = json.loads(body.decode('utf-8'))
except netfetch.FetchError as e:
raise ComputeError(f"LLM backend request failed: {e}")
except (ValueError, UnicodeDecodeError) as e:
raise ComputeError(f"LLM backend returned malformed JSON: {e}")
if not isinstance(resp_data, dict):
raise ComputeError("LLM backend returned a non-object response")
try:
raw_output_text, prompt_tokens, completion_tokens = provider.parse_response(
resp_data,
default_prompt_tokens=prompt_tokens,
)
except ValueError as e:
raise ComputeError(str(e))
gpu_seconds = round(time.time() - start_t, 3)
total_tokens = prompt_tokens + completion_tokens
rate_msat = provider.rate_msat_per_token or 12
fuel_used = total_tokens * rate_msat
if fuel_used > fuel_cap:
raise ComputeError(f"LLM generation exceeded fuel cap ({fuel_used} > {fuel_cap})")
on_token = job_payload.get('_on_token')
if callable(on_token) and raw_output_text:
# Emit token chunks to the streaming callback
chunk_size = 18
for idx in range(0, len(raw_output_text), chunk_size):
try:
on_token(raw_output_text[idx:idx + chunk_size])
except Exception:
break
raw_bytes = raw_output_text.encode('utf-8')
result_hash = crypto.sha256(raw_bytes)
counters = {
'prompt_tokens': prompt_tokens,
'completion_tokens': completion_tokens,
'total_tokens': total_tokens,
'gpu_seconds': gpu_seconds,
'rate_msat_per_token': rate_msat,
}
return ComputeExecutionResult(
result_hash=result_hash,
raw_output=raw_bytes,
fuel_used=fuel_used,
engine_id=self.engine_id,
wasi_profile_digest=f"llm:{model}",
counters=counters,
verification_mode=verif_mode,
)
from .bacalhau import BacalhauExecutor
from .agent import (
AgentExecutor,
AgentExecutionContext,
AgentToolProtocol,
AgentToolResult,
AgentDecision,
AgentBackendProtocol,
CalculatorTool,
Sha256Tool,
RegexSearchTool,
FetchBlobTool,
SubAskTool,
FinishTool,
ScriptedAgentBackend,
LlmAgentBackend,
)
__all__ = [
'LlmExecutor',
'BacalhauExecutor',
'AgentExecutor',
'AgentExecutionContext',
'AgentToolProtocol',
'AgentToolResult',
'AgentDecision',
'AgentBackendProtocol',
'CalculatorTool',
'Sha256Tool',
'RegexSearchTool',
'FetchBlobTool',
'SubAskTool',
'FinishTool',
'ScriptedAgentBackend',
'LlmAgentBackend',
'PROVIDER_PRESETS',
'ProviderPreset',
'LinkedProviderConfig',
'OperatorProviderRouter',
]